From 14c276fb6ba7bc8cf4307ba21b4de346608aa511 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 21 May 2020 21:19:00 +0200 Subject: [PATCH 001/211] require recent pkgconfig to avoid pytest dependency conflicts Signed-off-by: oleg.hoefling --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6906174d..d3739f76 100644 --- a/setup.py +++ b/setup.py @@ -417,7 +417,7 @@ def prepare_static_build_linux(self): sources.append(os.path.join(root, file)) pyxmlsec = Extension('xmlsec', sources=sources) -setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig', 'lxml>=3.8'] +setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] if sys.version_info < (3, 4): setup_reqs.append('pathlib2') From 6e62c074086fdcadfb952873dab4ac2ca95840fa Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 23 May 2020 14:22:41 +0200 Subject: [PATCH 002/211] set up correct python version on macos Signed-off-by: oleg.hoefling --- .github/workflows/macosx.yml | 6 +++++- .github/workflows/sdist.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index daf2c3dd..a7e42b70 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,9 +5,13 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + python: [2.7, 3.5, 3.6, 3.7, 3.8] steps: - uses: actions/checkout@v1 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 9c6e1282..9a7a4b4e 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.8 - name: Install build dependencies From 096dd0c8865b1da68a97919d96c46dbe536a3dbc Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sun, 24 May 2020 22:44:25 +0200 Subject: [PATCH 003/211] new function to set default max columns size for base64 encoding Signed-off-by: oleg.hoefling --- src/main.c | 35 +++++++++++++++++++++++++++++++++++ src/xmlsec/__init__.pyi | 6 +++++- tests/test_main.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/test_main.py diff --git a/src/main.c b/src/main.c index 85f457f2..b1f49180 100644 --- a/src/main.c +++ b/src/main.c @@ -14,6 +14,7 @@ #include #include #include +#include #define _PYXMLSEC_FREE_NONE 0 #define _PYXMLSEC_FREE_XMLSEC 1 @@ -127,6 +128,34 @@ static PyObject* PyXmlSec_PyEnableDebugOutput(PyObject *self, PyObject* args, Py Py_RETURN_NONE; } +static char PyXmlSec_PyBase64DefaultLineSize__doc__[] = \ + "base64_default_line_size(size = None)\n" + "Configures the default maximum columns size for base64 encoding.\n\n" + "If ``size`` is not given, this function returns the current default size, acting as a getter. " + "If ``size`` is given, a new value is applied and this function returns nothing, acting as a setter.\n" + ":param size: new default size value (optional)\n" + ":type size: :class:`int` or :data:`None`"; +static PyObject* PyXmlSec_PyBase64DefaultLineSize(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = { "size", NULL }; + PyObject *pySize = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O:base64_default_line_size", kwlist, &pySize)) { + return NULL; + } + if (pySize == NULL) { + return PyLong_FromLong(xmlSecBase64GetDefaultLineSize()); + } + int size = (int)PyLong_AsLong(pySize); + if (PyErr_Occurred()) { + return NULL; + } + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be positive"); + return NULL; + } + xmlSecBase64SetDefaultLineSize(size); + Py_RETURN_NONE; +} + static PyMethodDef PyXmlSec_MainMethods[] = { { "init", @@ -146,6 +175,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_VARARGS|METH_KEYWORDS, PyXmlSec_PyEnableDebugOutput__doc__ }, + { + "base64_default_line_size", + (PyCFunction)PyXmlSec_PyBase64DefaultLineSize, + METH_VARARGS|METH_KEYWORDS, + PyXmlSec_PyBase64DefaultLineSize__doc__ + }, {NULL, NULL} /* sentinel */ }; diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 540553c4..9ec147e9 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -1,5 +1,5 @@ import sys -from typing import AnyStr, IO, Iterable, Optional, Type, TypeVar, Union +from typing import AnyStr, IO, Iterable, Optional, Type, TypeVar, Union, overload from lxml.etree import _Element @@ -24,6 +24,10 @@ _K = TypeVar('_K', bound=Key) def enable_debug_trace(enabled: bool = ...) -> None: ... def init() -> None: ... def shutdown() -> None: ... +@overload +def base64_default_line_size() -> int: ... +@overload +def base64_default_line_size(size: int) -> None: ... class EncryptionContext: key: Optional[Key] diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 00000000..a0d144ca --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,32 @@ +import xmlsec +from tests import base + + +class TestBase64LineSize(base.TestMemoryLeaks): + def tearDown(self): + xmlsec.base64_default_line_size(64) + super(TestBase64LineSize, self).tearDown() + + def test_get_base64_default_line_size(self): + self.assertEqual(xmlsec.base64_default_line_size(), 64) + + def test_set_base64_default_line_size_positional_arg(self): + xmlsec.base64_default_line_size(0) + self.assertEqual(xmlsec.base64_default_line_size(), 0) + + def test_set_base64_default_line_size_keyword_arg(self): + xmlsec.base64_default_line_size(size=0) + self.assertEqual(xmlsec.base64_default_line_size(), 0) + + def test_set_base64_default_line_size_with_bad_args(self): + size = xmlsec.base64_default_line_size() + for bad_size in (None, '', object()): + with self.assertRaises(TypeError): + xmlsec.base64_default_line_size(bad_size) + self.assertEqual(xmlsec.base64_default_line_size(), size) + + def test_set_base64_default_line_size_rejects_negative_values(self): + size = xmlsec.base64_default_line_size() + with self.assertRaises(ValueError): + xmlsec.base64_default_line_size(-1) + self.assertEqual(xmlsec.base64_default_line_size(), size) From 7c73b28e6d332a14d3fce4ada754e54d01ed539d Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Mon, 25 May 2020 11:48:10 +0200 Subject: [PATCH 004/211] add source and documentation urls to package metadata Signed-off-by: oleg.hoefling --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d3739f76..10590135 100644 --- a/setup.py +++ b/setup.py @@ -442,6 +442,7 @@ def prepare_static_build_linux(self): maintainer='Oleg Hoefling', maintainer_email='oleg.hoefling@gmail.com', url='https://github.com/mehcode/python-xmlsec', + project_urls={'Documentation': 'https://xmlsec.readthedocs.io', 'Source': 'https://github.com/mehcode/python-xmlsec',}, license='MIT', keywords=['xmlsec'], classifiers=[ From 5531e0943d6e535db57a31a36c93cb9bef7edb06 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 28 May 2020 11:37:21 +0200 Subject: [PATCH 005/211] record and report code coverage in macosx jobs Signed-off-by: oleg.hoefling --- .github/workflows/macosx.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index a7e42b70..02c778d4 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -16,17 +16,29 @@ jobs: run: | pip install --upgrade pip setuptools wheel brew install libxml2 libxmlsec1 pkg-config + - name: Build macosx_x86_64 wheel + run: | + python setup.py bdist_wheel - name: Set environment variables shell: bash run: | echo ::set-env name=PKGVER::$(python setup.py --version) - - name: Build macosx_x86_64 wheel - run: | - python setup.py bdist_wheel + echo ::set-env name=LLVM_PROFILE_FILE::"pyxmlsec-%p.profraw" - name: Install test dependencies + env: + CC: clang + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" run: | - pip install --upgrade -r requirements-test.txt - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + rm -rf build/ + pip install coverage --upgrade -r requirements-test.txt + pip install --editable . - name: Run tests run: | - pytest -v --color=yes + coverage run -m pytest -v --color=yes + - name: Report coverage to codecov + run: | + LIBFILE=$(python -c "import xmlsec; print(xmlsec.__file__)") + /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse pyxmlsec-*.profraw -output pyxmlsec.profdata + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${LIBFILE} -instr-profile=pyxmlsec.profdata src > coverage.txt + bash <(curl -s https://codecov.io/bash) -f coverage.txt From 75cc6c093d323e672728fdd830ebb868f838c4f1 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 28 May 2020 14:21:59 +0200 Subject: [PATCH 006/211] also use codecov bash uploader in travis jobs since codecov-python has more issues to resolve Signed-off-by: oleg.hoefling --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5738884..4e9aa228 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,14 +35,14 @@ addons: - lcov install: - travis_retry pip install --upgrade pip setuptools wheel -- travis_retry pip install coverage codecov -r requirements-test.txt --upgrade --force-reinstall +- travis_retry pip install coverage -r requirements-test.txt --upgrade --force-reinstall - travis_retry pip install -e "." - pip list script: coverage run -m pytest -v tests --color=yes after_success: - lcov --capture --directory . --output-file coverage.info - lcov --list coverage.info -- codecov --file coverage.info +- bash <(curl -s https://codecov.io/bash) -f coverage.info before_deploy: - travis_retry pip install Sphinx -r doc/source/requirements.txt - git apply --verbose --no-index --unsafe-paths --directory=$(python -c "import site; print(site.getsitepackages()[0])") doc/source/sphinx-pr-6916.diff From f3bcec9aaa2ea49ea901b2fadfaf40b13db3772d Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 28 May 2020 16:36:10 +0200 Subject: [PATCH 007/211] initialize size at the beginning of the function Signed-off-by: oleg.hoefling --- src/main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.c b/src/main.c index b1f49180..59989529 100644 --- a/src/main.c +++ b/src/main.c @@ -138,13 +138,14 @@ static char PyXmlSec_PyBase64DefaultLineSize__doc__[] = \ static PyObject* PyXmlSec_PyBase64DefaultLineSize(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = { "size", NULL }; PyObject *pySize = NULL; + int size; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O:base64_default_line_size", kwlist, &pySize)) { return NULL; } if (pySize == NULL) { return PyLong_FromLong(xmlSecBase64GetDefaultLineSize()); } - int size = (int)PyLong_AsLong(pySize); + size = (int)PyLong_AsLong(pySize); if (PyErr_Occurred()) { return NULL; } From 0624740220430ee9a15aa930e4e72ea9a96e85d7 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 28 May 2020 16:42:53 +0200 Subject: [PATCH 008/211] publish test results in appveyor jobs Signed-off-by: oleg.hoefling --- .appveyor.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e3aee77a..939d8d30 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,5 +23,12 @@ test: off test_script: - pip install -r requirements-test.txt - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist - - pytest -v --color=yes + - pytest -v --color=yes --junitxml=unittests.xml - ps: Get-ChildItem dist\*.whl | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + +on_finish: + - ps: | + # archive test results at AppVeyor + $wc = New-Object 'System.Net.WebClient' + $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\unittests.xml)) + $LastExitCode = 0 From 2f62ff74614909d69440fe7fef53b299890e9fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Fri, 29 May 2020 22:59:29 +0200 Subject: [PATCH 009/211] Increase test coverage (#144) Work towards 100% code coverage in unit tests, fix segfaults encountered --- .travis.yml | 2 +- src/ds.c | 11 +++ src/enc.c | 11 ++- src/keys.c | 13 +++- tests/base.py | 3 + tests/conftest.py | 10 +++ tests/test_ds.py | 146 +++++++++++++++++++++++++++++++++++++-- tests/test_enc.py | 136 +++++++++++++++++++++++++++++++++--- tests/test_keys.py | 98 ++++++++++++++++++++++++-- tests/test_templates.py | 148 +++++++++++++++++++++++++++++----------- tests/test_tree.py | 20 +++++- tests/test_xmlsec.py | 14 ++++ 12 files changed, 548 insertions(+), 64 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_xmlsec.py diff --git a/.travis.yml b/.travis.yml index 4e9aa228..2abd7f82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ install: - pip list script: coverage run -m pytest -v tests --color=yes after_success: -- lcov --capture --directory . --output-file coverage.info +- lcov --capture --no-external --directory . --output-file coverage.info - lcov --list coverage.info - bash <(curl -s https://codecov.io/bash) -f coverage.info before_deploy: diff --git a/src/ds.c b/src/ds.c index 43d3d118..d0b4bdf9 100644 --- a/src/ds.c +++ b/src/ds.c @@ -87,11 +87,21 @@ static int PyXmlSec_SignatureContextKeySet(PyObject* self, PyObject* value, void PyXmlSec_Key* key; PYXMLSEC_DEBUGF("%p, %p", self, value); + + if (value == NULL) { // key deletion + if (ctx->handle->signKey != NULL) { + xmlSecKeyDestroy(ctx->handle->signKey); + ctx->handle->signKey = NULL; + } + return 0; + } + if (!PyObject_IsInstance(value, (PyObject*)PyXmlSec_KeyType)) { PyErr_SetString(PyExc_TypeError, "instance of *xmlsec.Key* expected."); return -1; } key = (PyXmlSec_Key*)value; + if (key->handle == NULL) { PyErr_SetString(PyExc_TypeError, "empty key."); return -1; @@ -252,6 +262,7 @@ static int PyXmlSec_ProcessSignBinary(PyXmlSec_SignatureContext* ctx, const xmlS if (ctx->handle->signKey == NULL) { PyErr_SetString(PyXmlSec_Error, "Sign key is not specified."); + return -1; } if (ctx->handle->signMethod != NULL) { diff --git a/src/enc.c b/src/enc.c index 02c01d8a..aaf35ae5 100644 --- a/src/enc.c +++ b/src/enc.c @@ -90,6 +90,15 @@ static int PyXmlSec_EncryptionContextKeySet(PyObject* self, PyObject* value, voi PyXmlSec_Key* key; PYXMLSEC_DEBUGF("%p, %p", self, value); + + if (value == NULL) { // key deletion + if (ctx->handle->encKey != NULL) { + xmlSecKeyDestroy(ctx->handle->encKey); + ctx->handle->encKey = NULL; + } + return 0; + } + if (!PyObject_IsInstance(value, (PyObject*)PyXmlSec_KeyType)) { PyErr_SetString(PyExc_TypeError, "instance of *xmlsec.Key* expected."); return -1; @@ -224,7 +233,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXml(PyObject* self, PyObject* } tmpType = xmlGetProp(template->_c_node, XSTR("Type")); if (tmpType == NULL || !(xmlStrEqual(tmpType, xmlSecTypeEncElement) || xmlStrEqual(tmpType, xmlSecTypeEncContent))) { - PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`)"); + PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`"); goto ON_FAIL; } diff --git a/src/keys.c b/src/keys.c index 7fd080a1..357cc9c7 100644 --- a/src/keys.c +++ b/src/keys.c @@ -452,10 +452,21 @@ static int PyXmlSec_KeyNameSet(PyObject* self, PyObject* value, void* closure) { return -1; } + if (value == NULL) { + if (xmlSecKeySetName(key->handle, value) < 0) { + PyXmlSec_SetLastError("cannot delete name"); + return -1; + } + return 0; + } + name = PyString_AsString(value); if (name == NULL) return -1; - xmlSecKeySetName(key->handle, XSTR(name)); + if (xmlSecKeySetName(key->handle, XSTR(name)) < 0) { + PyXmlSec_SetLastError("cannot set name"); + return -1; + } return 0; } diff --git a/tests/base.py b/tests/base.py index b05de1de..cf659b61 100644 --- a/tests/base.py +++ b/tests/base.py @@ -7,6 +7,9 @@ import unittest +if sys.version_info < (3, ): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + etype = type(etree.Element("test")) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..b51c4d45 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +def pytest_collection_modifyitems(items): + """ + Put the module init test first to implicitly check whether + any subsequent test fails because of module reinitialization. + """ + + def module_init_tests_first(item): + return int('test_xmlsec.py::TestModule::test_reinitialize_module' not in item.nodeid) + + items.sort(key=module_init_tests_first) diff --git a/tests/test_ds.py b/tests/test_ds.py index 26d49075..cf49ee71 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -1,7 +1,5 @@ -from tests import base - import xmlsec - +from tests import base consts = xmlsec.constants @@ -11,18 +9,69 @@ def test_init(self): ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) del ctx - def test_key(self): + def test_init_no_keys_manager(self): + ctx = xmlsec.SignatureContext() + del ctx + + def test_init_bad_args(self): + with self.assertRaisesRegex(TypeError, 'KeysManager required'): + xmlsec.SignatureContext(manager='foo') + + def test_no_key(self): ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) self.assertIsNone(ctx.key) + + def test_del_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) + def test_set_key_bad_type(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + with self.assertRaisesRegex(TypeError, r'instance of \*xmlsec.Key\* expected.'): + ctx.key = '' + + def test_set_invalid_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + with self.assertRaisesRegex(TypeError, 'empty key.'): + ctx.key = xmlsec.Key() + def test_register_id(self): ctx = xmlsec.SignatureContext() root = self.load_xml("sign_template.xml") sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, "Id") ctx.register_id(sign, "Id") + def test_register_id_bad_args(self): + ctx = xmlsec.SignatureContext() + with self.assertRaises(TypeError): + ctx.register_id('') + + def test_register_id_with_namespace_without_attribute(self): + ctx = xmlsec.SignatureContext() + root = self.load_xml("sign_template.xml") + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, "Id") + with self.assertRaisesRegex(xmlsec.Error, 'missing attribute.'): + ctx.register_id(sign, "Id", id_ns='foo') + + def test_sign_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.sign('') + + def test_sign_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(xmlsec.InternalError, 'failed to sign'): + ctx.sign(self.load_xml('sign1-in.xml')) + def test_sign_case1(self): """Should sign a pre-constructed template file using a key from a PEM file.""" root = self.load_xml("sign1-in.xml") @@ -134,6 +183,23 @@ def test_sign_case5(self): ctx.sign(sign) self.assertEqual(self.load_xml("sign5-out.xml"), root) + def test_sign_binary_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.sign_binary(bytes=1, transform='') + + def test_sign_binary_no_key(self): + ctx = xmlsec.SignatureContext() + with self.assertRaisesRegex(xmlsec.Error, 'Sign key is not specified.'): + ctx.sign_binary(bytes=b'', transform=consts.TransformRsaSha1) + + def test_sign_binary_invalid_signature_method(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(xmlsec.Error, 'incompatible signature method'): + ctx.sign_binary(bytes=b'', transform=consts.TransformXslt) + def test_sign_binary(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) @@ -144,6 +210,26 @@ def test_sign_binary(self): sign = ctx.sign_binary(self.load("sign6-in.bin"), consts.TransformRsaSha1) self.assertEqual(self.load("sign6-out.bin"), sign) + def test_sign_binary_twice_not_possible(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + data = self.load('sign6-in.bin') + ctx.sign_binary(data, consts.TransformRsaSha1) + with self.assertRaisesRegex(xmlsec.Error, 'Signature context already used; it is designed for one use only.'): + ctx.sign_binary(data, consts.TransformRsaSha1) + + def test_verify_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.verify('') + + def test_verify_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(xmlsec.InternalError, 'failed to verify'): + ctx.verify(self.load_xml('sign1-in.xml')) + def test_verify_case_1(self): self.check_verify(1) @@ -191,3 +277,55 @@ def test_validate_binary_sign_fail(self): self.assertEqual("rsakey.pem", ctx.key.name) with self.assertRaises(xmlsec.Error): ctx.verify_binary(self.load("sign6-in.bin"), consts.TransformRsaSha1, b"invalid") + + def test_enable_reference_transform(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.enable_reference_transform(consts.TransformRsaSha1) + + def test_enable_reference_transform_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.enable_reference_transform('') + with self.assertRaises(TypeError): + ctx.enable_reference_transform(0) + with self.assertRaises(TypeError): + ctx.enable_reference_transform(consts.KeyDataAes) + + def test_enable_signature_transform(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.enable_signature_transform(consts.TransformRsaSha1) + + def test_enable_signature_transform_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.enable_signature_transform('') + with self.assertRaises(TypeError): + ctx.enable_signature_transform(0) + with self.assertRaises(TypeError): + ctx.enable_signature_transform(consts.KeyDataAes) + + def test_set_enabled_key_data(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.set_enabled_key_data([consts.KeyDataAes]) + + def test_set_enabled_key_data_empty(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.set_enabled_key_data([]) + + def test_set_enabled_key_data_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + with self.assertRaises(TypeError): + ctx.set_enabled_key_data(0) + + def test_set_enabled_key_data_bad_list(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(TypeError, 'expected list of KeyData constants.'): + ctx.set_enabled_key_data('foo') diff --git a/tests/test_enc.py b/tests/test_enc.py index 2f8c9d74..141e5753 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -1,7 +1,10 @@ -from tests import base +import os +import tempfile -import xmlsec +from lxml import etree +import xmlsec +from tests import base consts = xmlsec.constants @@ -11,17 +14,48 @@ def test_init(self): ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) del ctx - def test_key(self): + def test_init_no_keys_manager(self): + ctx = xmlsec.EncryptionContext() + del ctx + + def test_init_bad_args(self): + with self.assertRaisesRegex(TypeError, 'KeysManager required'): + xmlsec.EncryptionContext(manager='foo') + + def test_no_key(self): + ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) + self.assertIsNone(ctx.key) + + def test_get_key(self): ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) self.assertIsNone(ctx.key) ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) self.assertIsNotNone(ctx.key) + def test_del_key(self): + ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) + self.assertIsNotNone(ctx.key) + + def test_set_key_bad_type(self): + ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) + with self.assertRaisesRegex(TypeError, r'instance of \*xmlsec.Key\* expected.'): + ctx.key = '' + + def test_set_invalid_key(self): + ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) + with self.assertRaisesRegex(TypeError, 'empty key.'): + ctx.key = xmlsec.Key() + def test_encrypt_xml(self): root = self.load_xml('enc1-in.xml') - enc_data = xmlsec.template.encrypted_data_create( - root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns="xenc" - ) + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns="xenc") xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) @@ -49,6 +83,30 @@ def test_encrypt_xml(self): cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) + def test_encrypt_xml_bad_args(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaises(TypeError): + ctx.encrypt_xml('', 0) + + def test_encrypt_xml_bad_template(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaisesRegex(xmlsec.Error, 'unsupported `Type`, it should be `element` or `content`'): + ctx.encrypt_xml(etree.Element('root'), etree.Element('node')) + + def test_encrypt_xml_bad_template_bad_type_attribute(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaisesRegex(xmlsec.Error, 'unsupported `Type`, it should be `element` or `content`'): + root = etree.Element('root') + root.attrib['Type'] = 'foo' + ctx.encrypt_xml(root, etree.Element('node')) + + def test_encrypt_xml_fail(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaisesRegex(xmlsec.Error, 'failed to encrypt xml'): + root = etree.Element('root') + root.attrib['Type'] = consts.TypeEncElement + ctx.encrypt_xml(root, etree.Element('node')) + def test_encrypt_binary(self): root = self.load_xml('enc2-in.xml') enc_data = xmlsec.template.encrypted_data_create( @@ -81,6 +139,61 @@ def test_encrypt_binary(self): cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) + def test_encrypt_binary_bad_args(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaises(TypeError): + ctx.encrypt_binary('', 0) + + def test_encrypt_binary_bad_template(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaisesRegex(xmlsec.Error, 'failed to encrypt binary'): + ctx.encrypt_binary(etree.Element('root'), b'data') + + def test_encrypt_uri(self): + root = self.load_xml('enc2-in.xml') + enc_data = xmlsec.template.encrypted_data_create( + root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns="xenc", mime_type="binary/octet-stream" + ) + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") + ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) + xmlsec.template.encrypted_data_ensure_cipher_value(ek) + + manager = xmlsec.KeysManager() + manager.add_key(xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem)) + + ctx = xmlsec.EncryptionContext(manager) + ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + tmpfile.write(b'test') + + encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) + self.assertIsNotNone(encrypted) + self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + + enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) + self.assertIsNotNone(enc_method) + self.assertEqual("http://www.w3.org/2001/04/xmlenc#aes128-cbc", enc_method.get("Algorithm")) + + ki = xmlsec.tree.find_child(enc_data, consts.NodeKeyInfo, consts.DSigNs) + self.assertIsNotNone(ki) + enc_method2 = xmlsec.tree.find_node(ki, consts.NodeEncryptionMethod, consts.EncNs) + self.assertIsNotNone(enc_method2) + self.assertEqual("http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p", enc_method2.get("Algorithm")) + cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) + self.assertIsNotNone(cipher_value) + + def test_encrypt_uri_bad_args(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaises(TypeError): + ctx.encrypt_uri('', 0) + + def test_encrypt_uri_fail(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaisesRegex(xmlsec.InternalError, 'failed to encrypt URI'): + ctx.encrypt_uri(etree.Element('root'), '') + def test_decrypt1(self): self.check_decrypt(1) @@ -117,18 +230,21 @@ def check_decrypt(self, i): self.assertIsNotNone(decrypted) self.assertEqual(self.load_xml("enc%d-in.xml" % i), root) + def test_decrypt_bad_args(self): + ctx = xmlsec.EncryptionContext() + with self.assertRaises(TypeError): + ctx.decrypt('') def check_no_segfault(self): - namespaces = { - 'soap': 'http://schemas.xmlsoap.org/soap/envelope/' - } + namespaces = {'soap': 'http://schemas.xmlsoap.org/soap/envelope/'} manager = xmlsec.KeysManager() key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) manager.add_key(key) template = self.load_xml('enc-bad-in.xml') enc_data = xmlsec.template.encrypted_data_create( - template, xmlsec.Transform.AES128, type=xmlsec.EncryptionType.CONTENT, ns='xenc') + template, xmlsec.Transform.AES128, type=xmlsec.EncryptionType.CONTENT, ns='xenc' + ) xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') enc_key = xmlsec.template.add_encrypted_key(key_info, xmlsec.Transform.RSA_PKCS1) diff --git a/tests/test_keys.py b/tests/test_keys.py index add11e41..12b8224f 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -1,9 +1,8 @@ -from tests import base - import copy +import tempfile import xmlsec - +from tests import base consts = xmlsec.constants @@ -17,6 +16,10 @@ def test_key_from_memory_with_bad_args(self): with self.assertRaises(TypeError): xmlsec.Key.from_memory(1, format="") + def test_key_from_memory_invalid_data(self): + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load key.*'): + xmlsec.Key.from_memory(b'foo', format=consts.KeyDataFormatPem) + def test_key_from_file(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) @@ -25,11 +28,23 @@ def test_key_from_file_with_bad_args(self): with self.assertRaises(TypeError): xmlsec.Key.from_file(1, format="") + def test_key_from_invalid_file(self): + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + xmlsec.Key.from_file(tmpfile.name, format=consts.KeyDataFormatPem) + def test_key_from_fileobj(self): with open(self.path("rsakey.pem"), "rb") as fobj: key = xmlsec.Key.from_file(fobj, format=consts.KeyDataFormatPem) self.assertIsNotNone(key) + def test_key_from_invalid_fileobj(self): + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + tmpfile.write(b'foo') + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'), open(tmpfile.name) as fp: + xmlsec.Key.from_file(fp, format=consts.KeyDataFormatPem) + def test_generate(self): key = xmlsec.Key.generate(klass=consts.KeyDataAes, size=256, type=consts.KeyDataTypeSession) self.assertIsNotNone(key) @@ -38,6 +53,10 @@ def test_generate_with_bad_args(self): with self.assertRaises(TypeError): xmlsec.Key.generate(klass="", size="", type="") + def test_generate_invalid_size(self): + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot generate key.*'): + xmlsec.Key.generate(klass=consts.KeyDataAes, size=0, type=consts.KeyDataTypeSession) + def test_from_binary_file(self): key = xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=self.path("deskey.bin")) self.assertIsNotNone(key) @@ -46,6 +65,12 @@ def test_from_binary_file_with_bad_args(self): with self.assertRaises(TypeError): xmlsec.Key.from_binary_file(klass="", filename=1) + def test_from_invalid_binary_file(self): + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=tmpfile.name) + def test_from_binary_data(self): key = xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=self.load("deskey.bin")) self.assertIsNotNone(key) @@ -54,6 +79,10 @@ def test_from_binary_data_with_bad_args(self): with self.assertRaises(TypeError): xmlsec.Key.from_binary_data(klass="", data=1) + def test_from_invalid_binary_data(self): + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=b'') + def test_load_cert_from_file(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) @@ -65,12 +94,34 @@ def test_load_cert_from_file_with_bad_args(self): with self.assertRaises(TypeError): key.load_cert_from_file(1, format="") + def test_load_cert_from_invalid_file(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + self.assertIsNotNone(key) + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + key.load_cert_from_file(tmpfile.name, format=consts.KeyDataFormatPem) + def test_load_cert_from_fileobj(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) with open(self.path("rsacert.pem"), "rb") as fobj: key.load_cert_from_file(fobj, format=consts.KeyDataFormatPem) + def test_load_cert_from_fileobj_with_bad_args(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + self.assertIsNotNone(key) + with self.assertRaises(TypeError), open(self.path("rsacert.pem"), "rb") as fobj: + key.load_cert_from_file(fobj, format='') + + def test_load_cert_from_invalid_fileobj(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + self.assertIsNotNone(key) + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + tmpfile.write(b'foo') + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'), open(tmpfile.name) as fp: + key.load_cert_from_file(fp, format=consts.KeyDataFormatPem) + def test_load_cert_from_memory(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) @@ -82,12 +133,37 @@ def test_load_cert_from_memory_with_bad_args(self): with self.assertRaises(TypeError): key.load_cert_from_memory(1, format="") - def test_name(self): + def test_load_cert_from_memory_invalid_data(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + self.assertIsNotNone(key) + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + key.load_cert_from_memory(b'', format=consts.KeyDataFormatPem) + + def test_get_name(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + self.assertIsNone(key.name) + + def test_get_name_invalid_key(self): + key = xmlsec.Key() + with self.assertRaisesRegex(ValueError, 'key is not ready'): + key.name + + def test_del_name(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key.name = "rsakey" + del key.name self.assertIsNone(key.name) + + def test_set_name(self): + key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) key.name = "rsakey" self.assertEqual("rsakey", key.name) + def test_set_name_invalid_key(self): + key = xmlsec.Key() + with self.assertRaisesRegex(ValueError, 'key is not ready'): + key.name = 'foo' + def test_copy(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) key2 = copy.copy(key) @@ -112,6 +188,14 @@ def test_load_cert(self): mngr.load_cert(self.path("rsacert.pem"), format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) def test_load_cert_with_bad_args(self): + mngr = xmlsec.KeysManager() + mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + mngr.load_cert(tmpfile.name, format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) + + def test_load_invalid_cert(self): mngr = xmlsec.KeysManager() mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) with self.assertRaises(TypeError): @@ -128,6 +212,12 @@ def test_load_cert_from_memory_with_bad_args(self): with self.assertRaises(TypeError): mngr.load_cert_from_memory(1, format="", type="") + def test_load_cert_from_memory_invalid_data(self): + mngr = xmlsec.KeysManager() + mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + mngr.load_cert_from_memory(b'', format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) + def test_load_invalid_key(self): mngr = xmlsec.KeysManager() with self.assertRaises(ValueError): diff --git a/tests/test_templates.py b/tests/test_templates.py index dcd8d940..947e0bbc 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,7 +1,7 @@ -from tests import base +from lxml import etree import xmlsec - +from tests import base consts = xmlsec.constants @@ -10,20 +10,19 @@ class TestTemplates(base.TestMemoryLeaks): def test_create(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create( - root, - c14n_method=consts.TransformExclC14N, - sign_method=consts.TransformRsaSha1, - id="Id", - ns="test" + root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1, id="Id", ns="test" ) self.assertEqual("Id", sign.get("Id")) self.assertEqual("test", sign.prefix) + def test_create_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.create('', c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + def test_encrypt_data_create(self): root = self.load_xml("doc.xml") enc = xmlsec.template.encrypted_data_create( - root, method=consts.TransformDes3Cbc, id="Id", type="Type", mime_type="MimeType", encoding="Encoding", - ns="test" + root, method=consts.TransformDes3Cbc, id="Id", type="Type", mime_type="MimeType", encoding="Encoding", ns="test" ) for a in ("Id", "Type", "MimeType", "Encoding"): self.assertEqual(a, enc.get(a)) @@ -35,18 +34,21 @@ def test_ensure_key_info(self): ki = xmlsec.template.ensure_key_info(sign, id="Id") self.assertEqual("Id", ki.get("Id")) + def test_ensure_key_info_fail(self): + with self.assertRaisesRegex(xmlsec.InternalError, 'cannot ensure key info.'): + xmlsec.template.ensure_key_info(etree.fromstring(b''), id="Id") + + def test_ensure_key_info_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.ensure_key_info('', id=0) + def test_add_encrypted_key(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) - self.assertEqual( - ek, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeEncryptedKey, consts.EncNs) - ) - ek2 = xmlsec.template.add_encrypted_key( - ki, consts.TransformRsaOaep, id="Id", type="Type", recipient="Recipient" - ) + self.assertEqual(ek, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeEncryptedKey, consts.EncNs)) + ek2 = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep, id="Id", type="Type", recipient="Recipient") for a in ("Id", "Type", "Recipient"): self.assertEqual(a, ek2.get(a)) @@ -55,13 +57,22 @@ def test_add_key_name(self): sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) kn = xmlsec.template.add_key_name(ki) - self.assertEqual( - kn, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyName, consts.DSigNs) - ) + self.assertEqual(kn, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyName, consts.DSigNs)) kn2 = xmlsec.template.add_key_name(ki, name="name") self.assertEqual("name", kn2.text) + def test_add_key_name_none(self): + root = self.load_xml("doc.xml") + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + ki = xmlsec.template.ensure_key_info(sign) + kn2 = xmlsec.template.add_key_name(ki, name=None) + self.assertEqual(kn2.text, None) + print(etree.tostring(kn2)) + + def test_add_key_name_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_key_name('') + def test_add_reference(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) @@ -69,15 +80,32 @@ def test_add_reference(self): for a in ("Id", "URI", "Type"): self.assertEqual(a, ref.get(a)) + def test_add_reference_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_reference('', consts.TransformSha1) + with self.assertRaises(TypeError): + xmlsec.template.add_reference(etree.Element('root'), '') + + def test_add_reference_fail(self): + with self.assertRaisesRegex(xmlsec.InternalError, 'cannot add reference.'): + xmlsec.template.add_reference(etree.Element('root'), consts.TransformSha1) + + def test_add_transform_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_transform('', consts.TransformSha1) + with self.assertRaises(TypeError): + xmlsec.template.add_transform(etree.Element('root'), '') + def test_add_key_value(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) kv = xmlsec.template.add_key_value(ki) - self.assertEqual( - kv, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyValue, consts.DSigNs) - ) + self.assertEqual(kv, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyValue, consts.DSigNs)) + + def test_add_key_value_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_key_value('') def test_add_x509_data(self): root = self.load_xml("doc.xml") @@ -91,10 +119,11 @@ def test_add_x509_data(self): xmlsec.template.x509_data_add_subject_name(x509) xmlsec.template.x509_issuer_serial_add_issuer_name(issuer) xmlsec.template.x509_issuer_serial_add_serial_number(issuer) - self.assertEqual( - x509, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeX509Data, consts.DSigNs) - ) + self.assertEqual(x509, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeX509Data, consts.DSigNs)) + + def test_add_x509_data_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_x509_data('') def test_x509_issuer_serial_add_issuer(self): root = self.load_xml("doc.xml") @@ -107,27 +136,65 @@ def test_x509_issuer_serial_add_issuer(self): self.assertEqual("Name", name.text) self.assertEqual("Serial", serial.text) + def test_x509_issuer_serial_add_issuer_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_data_add_issuer_serial('') + + def test_x509_issuer_serial_add_issuer_name_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_issuer_serial_add_issuer_name('') + + def test_x509_issuer_serial_add_serial_number_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_issuer_serial_add_serial_number('') + + def test_x509_data_add_subject_name_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_data_add_subject_name('') + + def test_x509_data_add_ski_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_data_add_ski('') + + def test_x509_data_add_certificate_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_data_add_certificate('') + + def test_x509_data_add_crl_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.x509_data_add_crl('') + + def test_add_encrypted_key_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.add_encrypted_key('', 0) + + def test_encrypted_data_create_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.encrypted_data_create('', 0) + def test_encrypted_data_ensure_cipher_value(self): root = self.load_xml("doc.xml") enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc) cv = xmlsec.template.encrypted_data_ensure_cipher_value(enc) - self.assertEqual( - cv, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeCipherValue, consts.EncNs) - ) + self.assertEqual(cv, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeCipherValue, consts.EncNs)) + + def test_encrypted_data_ensure_cipher_value_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.encrypted_data_ensure_cipher_value('') def test_encrypted_data_ensure_key_info(self): root = self.load_xml("doc.xml") enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc) ki = xmlsec.template.encrypted_data_ensure_key_info(enc) - self.assertEqual( - ki, - xmlsec.tree.find_node(self.load_xml("enc_template.xml"), consts.NodeKeyInfo, consts.DSigNs) - ) + self.assertEqual(ki, xmlsec.tree.find_node(self.load_xml("enc_template.xml"), consts.NodeKeyInfo, consts.DSigNs)) ki2 = xmlsec.template.encrypted_data_ensure_key_info(enc, id="Id", ns="test") self.assertEqual("Id", ki2.get("Id")) self.assertEqual("test", ki2.prefix) + def test_encrypted_data_ensure_key_info_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.encrypted_data_ensure_key_info('') + def test_transform_add_c14n_inclusive_namespaces(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) @@ -136,7 +203,8 @@ def test_transform_add_c14n_inclusive_namespaces(self): xmlsec.template.transform_add_c14n_inclusive_namespaces(trans1, "default") trans2 = xmlsec.template.add_transform(ref, consts.TransformXslt) xmlsec.template.transform_add_c14n_inclusive_namespaces(trans2, ["ns1", "ns2"]) - self.assertEqual( - ref, - xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeReference, consts.DSigNs) - ) + self.assertEqual(ref, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeReference, consts.DSigNs)) + + def test_transform_add_c14n_inclusive_namespaces_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.template.transform_add_c14n_inclusive_namespaces('', []) diff --git a/tests/test_tree.py b/tests/test_tree.py index f07c654c..4c79c8de 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,7 +1,5 @@ -from tests import base - import xmlsec - +from tests import base consts = xmlsec.constants @@ -14,18 +12,34 @@ def test_find_child(self): self.assertIsNone(xmlsec.tree.find_child(root, consts.NodeReference)) self.assertIsNone(xmlsec.tree.find_child(root, consts.NodeSignedInfo, consts.EncNs)) + def test_find_child_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.tree.find_child('', 0, True) + def test_find_parent(self): root = self.load_xml("sign_template.xml") si = xmlsec.tree.find_child(root, consts.NodeSignedInfo, consts.DSigNs) self.assertIs(root, xmlsec.tree.find_parent(si, consts.NodeSignature)) self.assertIsNone(xmlsec.tree.find_parent(root, consts.NodeSignedInfo)) + def test_find_parent_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.tree.find_parent('', 0, True) + def test_find_node(self): root = self.load_xml("sign_template.xml") ref = xmlsec.tree.find_node(root, consts.NodeReference) self.assertEqual(consts.NodeReference, ref.tag.partition('}')[2]) self.assertIsNone(xmlsec.tree.find_node(root, consts.NodeReference, consts.EncNs)) + def test_find_node_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.tree.find_node('', 0, True) + def test_add_ids(self): root = self.load_xml("sign_template.xml") xmlsec.tree.add_ids(root, ["id1", "id2", "id3"]) + + def test_add_ids_bad_args(self): + with self.assertRaises(TypeError): + xmlsec.tree.add_ids('', []) diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py new file mode 100644 index 00000000..32fac69a --- /dev/null +++ b/tests/test_xmlsec.py @@ -0,0 +1,14 @@ +import xmlsec +from tests import base + + +class TestModule(base.TestMemoryLeaks): + def test_reinitialize_module(self): + """ + This doesn't explicitly test anything, but will + be invoked first in the suite, so if the subsequent + tests don't fail, we know that the ``init()``/``shutdown()`` + function pair doesn't break anything. + """ + xmlsec.shutdown() + xmlsec.init() From 39c3bca3ce0ea55a8b29ce8a0999bfb4a72abb64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Wed, 10 Jun 2020 17:51:35 +0100 Subject: [PATCH 010/211] add custom prefix to environment variables in setup script (#153) Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 2 +- .github/workflows/sdist.yml | 2 +- setup.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 7d58279a..7ed19a5c 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -18,7 +18,7 @@ jobs: echo ::set-env name=PKGVER::$(/opt/python/${{ matrix.python-abi }}/bin/python setup.py --version) - name: Build linux_x86_64 wheel env: - STATIC_DEPS: true + PYXMLSEC_STATIC_DEPS: true run: | /opt/python/${{ matrix.python-abi }}/bin/python setup.py bdist_wheel - name: Label manylinux2010_x86_64 wheel diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 9a7a4b4e..281cdaf9 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -17,7 +17,7 @@ jobs: python setup.py sdist - name: Install test dependencies env: - STATIC_DEPS: true + PYXMLSEC_STATIC_DEPS: true run: | pip install --upgrade -r requirements-test.txt pip install dist/xmlsec-$(python setup.py --version).tar.gz diff --git a/setup.py b/setup.py index 10590135..976489c6 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,8 @@ def run(self): from pathlib2 import Path ext = self.ext_map['xmlsec'] - self.debug = os.environ.get('DEBUG', False) - self.static = os.environ.get('STATIC_DEPS', False) + self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) + self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) if self.static or sys.platform == 'win32': self.info('starting static build on {}'.format(sys.platform)) @@ -43,7 +43,7 @@ def run(self): self.build_libs_dir = buildroot / 'libs' self.build_libs_dir.mkdir(exist_ok=True) - self.libs_dir = Path(os.environ.get('LIBS_DIR', 'libs')) + self.libs_dir = Path(os.environ.get('PYXMLSEC_LIBS_DIR', 'libs')) self.libs_dir.mkdir(exist_ok=True) if sys.platform == 'win32': @@ -180,12 +180,12 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('OPENSSL_VERSION', '1.1.1g') - self.libiconv_version = os.environ.get('LIBICONV_VERSION', '1.16') - self.libxml2_version = os.environ.get('LIBXML2_VERSION', None) - self.libxslt_version = os.environ.get('LIBXLST_VERSION', None) - self.zlib_version = os.environ.get('ZLIB_VERSION', '1.2.11') - self.xmlsec1_version = os.environ.get('XMLSEC1_VERSION', '1.2.30') + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1g') + self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION', '1.16') + self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', None) + self.libxslt_version = os.environ.get('PYXMLSEC_LIBXLST_VERSION', None) + self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.2.11') + self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.2.30') self.info('Settings:') self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) From 78958c8096a00f01c2a84a4bf7084a2d1fb04352 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 21 May 2020 21:36:04 +0200 Subject: [PATCH 011/211] add aes-gcm transforms Signed-off-by: oleg.hoefling --- src/constants.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/constants.c b/src/constants.c index 02f5250c..68f086e9 100644 --- a/src/constants.c +++ b/src/constants.c @@ -543,6 +543,12 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformSha384, "SHA384"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformSha512, "SHA512"); +#if XMLSEC_VERSION_HEX > 315 + PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes128Gcm, "AES128_GCM"); + PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes192Gcm, "AES192_GCM"); + PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes256Gcm, "AES256_GCM"); +#endif + PYXMLSEC_CLOSE_NAMESPACE(transformCls); #undef PYXMLSEC_ADD_TRANSFORM_CONSTANT From 2a6cd997c53d56c3128aa603d4c7bef28d5dd6e8 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 2 Jul 2020 17:12:42 +0200 Subject: [PATCH 012/211] add typing constants Signed-off-by: oleg.hoefling --- src/xmlsec/constants.pyi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 71b717a7..8be27b23 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -79,8 +79,11 @@ NsExcC14NWithComments: Final = 'http://www.w3.org/2001/10/xml-exc-c14n#WithComme Soap11Ns: Final = 'http://schemas.xmlsoap.org/soap/envelope/' Soap12Ns: Final = 'http://www.w3.org/2002/06/soap-envelope' TransformAes128Cbc: Final = __Transform('aes128-cbc', 'http://www.w3.org/2001/04/xmlenc#aes128-cbc', 16) +TransformAes128Gcm: Final = __Transform('aes128-gcm', 'http://www.w3.org/2009/xmlenc11#aes128-gcm', 16) TransformAes192Cbc: Final = __Transform('aes192-cbc', 'http://www.w3.org/2001/04/xmlenc#aes192-cbc', 16) +TransformAes192Gcm: Final = __Transform('aes192-gcm', 'http://www.w3.org/2009/xmlenc11#aes192-gcm', 16) TransformAes256Cbc: Final = __Transform('aes256-cbc', 'http://www.w3.org/2001/04/xmlenc#aes256-cbc', 16) +TransformAes256Gcm: Final = __Transform('aes256-gcm', 'http://www.w3.org/2009/xmlenc11#aes256-gcm', 16) TransformDes3Cbc: Final = __Transform('tripledes-cbc', 'http://www.w3.org/2001/04/xmlenc#tripledes-cbc', 16) TransformDsaSha1: Final = __Transform('dsa-sha1', 'http://www.w3.org/2000/09/xmldsig#dsa-sha1', 8) TransformEcdsaSha1: Final = __Transform('ecdsa-sha1', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha1', 8) From 658001ef77ea7086c8391f2e0409072ed5bb9c60 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 2 Jul 2020 17:13:35 +0200 Subject: [PATCH 013/211] use badges with logo from shields.io Signed-off-by: oleg.hoefling --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index b58c518b..f01d467d 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,11 @@ python-xmlsec ============= -.. image:: https://travis-ci.org/mehcode/python-xmlsec.png?branch=master +.. image:: https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white + :target: https://pypi.python.org/pypi/xmlsec +.. image:: https://img.shields.io/travis/com/mehcode/python-xmlsec/master.svg?logo=travis&logoColor=white&label=Travis%20CI :target: https://travis-ci.org/mehcode/python-xmlsec -.. image:: https://ci.appveyor.com/api/projects/status/ij87xk5wo8a39jua?svg=true +.. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor :target: https://ci.appveyor.com/project/hoefling/xmlsec .. image:: https://github.com/mehcode/python-xmlsec/workflows/manylinux2010/badge.svg :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22manylinux2010%22 @@ -11,9 +13,7 @@ python-xmlsec :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22MacOS%22 .. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg :target: https://codecov.io/gh/mehcode/python-xmlsec -.. image:: https://img.shields.io/pypi/v/xmlsec.svg - :target: https://pypi.python.org/pypi/xmlsec -.. image:: https://readthedocs.org/projects/xmlsec/badge/?version=latest +.. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs :target: https://xmlsec.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status From ff32478725e80643b3ce0087665e55f0c9988b35 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 29 Oct 2020 13:51:26 +0100 Subject: [PATCH 014/211] add python 3.9 jobs Signed-off-by: oleg.hoefling --- .appveyor.yml | 2 ++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux2010.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 939d8d30..d945472a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -10,6 +10,8 @@ environment: - python: 37-x64 - python: 38 - python: 38-x64 + - python: 39 + - python: 39-x64 install: - SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH% diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 02c778d4..c8bfee95 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8] + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v1 - name: Setup Python diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 7ed19a5c..637983d4 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -6,7 +6,7 @@ jobs: container: quay.io/pypa/manylinux2010_x86_64 strategy: matrix: - python-abi: [cp27-cp27m, cp27-cp27mu, cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38] + python-abi: [cp27-cp27m, cp27-cp27mu, cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 281cdaf9..5b5dac96 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 2abd7f82..7297721e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: - python: 3.8 dist: xenial sudo: required - - python: 3.9-dev + - python: 3.9 dist: xenial sudo: required env: From 0275bbe33d58675556d1eb15121350b41313f083 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 29 Oct 2020 14:14:34 +0100 Subject: [PATCH 015/211] fix 3.9 not used in appveyor jobs Signed-off-by: oleg.hoefling --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index d945472a..c8beace3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,7 +11,9 @@ environment: - python: 38 - python: 38-x64 - python: 39 + image: Visual Studio 2019 - python: 39-x64 + image: Visual Studio 2019 install: - SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH% From 5a21f3ff5700c51f3e746d01ff8fd757f8f5c37a Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 29 Oct 2020 16:31:07 +0100 Subject: [PATCH 016/211] install python 3.9 via windows installer on appveyor Signed-off-by: oleg.hoefling --- .appveyor.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c8beace3..a36ddda1 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,11 +11,31 @@ environment: - python: 38 - python: 38-x64 - python: 39 - image: Visual Studio 2019 + python_version: 3.9.0 - python: 39-x64 - image: Visual Studio 2019 + python_version: 3.9.0 install: + - ps: | + # from https://github.com/appveyor/build-images/blob/27bde614bc60d7ef7a8bc46182f4d7582fa11b56/scripts/Windows/install_python.ps1#L88-L108 + function InstallPythonEXE($targetPath, $version) { + $urlPlatform = "" + if ($targetPath -match '-x64$') { + $urlPlatform = "-amd64" + } + Write-Host "Installing Python $version$urlPlatform to $($targetPath)..." -ForegroundColor Cyan + $downloadUrl = "https://www.python.org/ftp/python/$version/python-$version$urlPlatform.exe" + Write-Host "Downloading $($downloadUrl)..." + $exePath = "$env:TEMP\python-$version.exe" + (New-Object Net.WebClient).DownloadFile($downloadUrl, $exePath) + Write-Host "Installing..." + cmd /c start /wait $exePath /quiet TargetDir="$targetPath" Shortcuts=0 Include_launcher=1 InstallLauncherAllUsers=1 Include_debug=1 + Remove-Item $exePath + Write-Host "Installed Python $version" -ForegroundColor Green + } + if ( -not ( Test-Path -Path C:\\Python$env:PYTHON -PathType Container ) ) { + InstallPythonEXE C:\\Python$env:PYTHON $env:PYTHON_VERSION + } - SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH% - python -m pip install -U pip wheel setuptools From b7e940872c318de4fb3a4520c67812e7585df1b9 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 30 Oct 2020 00:29:45 +0100 Subject: [PATCH 017/211] install from wheels in ci to run tests over real installation Signed-off-by: oleg.hoefling --- .github/workflows/macosx.yml | 2 +- .travis.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index c8bfee95..6851af3a 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -32,7 +32,7 @@ jobs: run: | rm -rf build/ pip install coverage --upgrade -r requirements-test.txt - pip install --editable . + pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - name: Run tests run: | coverage run -m pytest -v --color=yes diff --git a/.travis.yml b/.travis.yml index 7297721e..4e403f90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,8 @@ addons: install: - travis_retry pip install --upgrade pip setuptools wheel - travis_retry pip install coverage -r requirements-test.txt --upgrade --force-reinstall -- travis_retry pip install -e "." -- pip list +- python setup.py bdist_wheel +- pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ script: coverage run -m pytest -v tests --color=yes after_success: - lcov --capture --no-external --directory . --output-file coverage.info From d5f461dae335292508eaef538509e23d4c97fc07 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 13 Nov 2020 22:08:02 +0100 Subject: [PATCH 018/211] don't build for python 2.7 Signed-off-by: oleg.hoefling --- .appveyor.yml | 2 -- .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux2010.yml | 2 +- .travis.yml | 1 - setup.py | 55 ++++++++--------------------- 5 files changed, 16 insertions(+), 46 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index a36ddda1..54bd2b1e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,7 +1,5 @@ environment: matrix: - - python: 27 - - python: 27-x64 - python: 35 - python: 35-x64 - python: 36 diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 6851af3a..01302083 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v1 - name: Setup Python diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 637983d4..2190378b 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -6,7 +6,7 @@ jobs: container: quay.io/pypa/manylinux2010_x86_64 strategy: matrix: - python-abi: [cp27-cp27m, cp27-cp27mu, cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] + python-abi: [cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.travis.yml b/.travis.yml index 4e403f90..9106805a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ notifications: email: false matrix: include: - - python: 2.7 - python: 3.5 - python: 3.6 - python: 3.7 diff --git a/setup.py b/setup.py index 976489c6..2838fe84 100644 --- a/setup.py +++ b/setup.py @@ -7,27 +7,18 @@ import zipfile from distutils import log from distutils.errors import DistutilsError +from pathlib import Path +from urllib.request import urlcleanup, urljoin, urlretrieve from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig -if sys.version_info >= (3, 4): - from urllib.request import urlcleanup, urljoin, urlretrieve -else: - from urllib import urlcleanup, urlretrieve - from urlparse import urljoin - class build_ext(build_ext_orig, object): def info(self, message): self.announce(message, level=log.INFO) def run(self): - if sys.version_info >= (3, 4): - from pathlib import Path - else: - from pathlib2 import Path - ext = self.ext_map['xmlsec'] self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) @@ -107,16 +98,10 @@ def run(self): def prepare_static_build_win(self): release_url = 'https://github.com/bgaifullin/libxml2-win-binaries/releases/download/v2018.08/' - if sys.version_info < (3, 5): - if sys.maxsize > 2147483647: - suffix = 'vs2008.win64' - else: - suffix = "vs2008.win32" + if sys.maxsize > 2147483647: + suffix = 'win64' else: - if sys.maxsize > 2147483647: - suffix = "win64" - else: - suffix = "win32" + suffix = 'win32' libs = [ 'libxml2-2.9.4.{}.zip'.format(suffix), @@ -185,7 +170,7 @@ def prepare_static_build_linux(self): self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', None) self.libxslt_version = os.environ.get('PYXMLSEC_LIBXLST_VERSION', None) self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.2.11') - self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.2.30') + self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.2.31') self.info('Settings:') self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) @@ -402,26 +387,11 @@ def prepare_static_build_linux(self): ext.extra_objects = [str(self.prefix_dir / 'lib' / o) for o in extra_objects] -if sys.version_info >= (3, 4): - from pathlib import Path - - src_root = Path(__file__).parent / 'src' - sources = [str(p.absolute()) for p in src_root.rglob('*.c')] -else: - import fnmatch - - src_root = os.path.join(os.path.dirname(__file__), 'src') - sources = [] - for root, _, files in os.walk(src_root): - for file in fnmatch.filter(files, '*.c'): - sources.append(os.path.join(root, file)) - +src_root = Path(__file__).parent / 'src' +sources = [str(p.absolute()) for p in src_root.rglob('*.c')] pyxmlsec = Extension('xmlsec', sources=sources) setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] -if sys.version_info < (3, 4): - setup_reqs.append('pathlib2') - with io.open('README.rst', encoding='utf-8') as f: long_desc = f.read() @@ -434,7 +404,7 @@ def prepare_static_build_linux(self): long_description=long_desc, ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires='>=3.5', setup_requires=setup_reqs, install_requires=['lxml>=3.8'], author="Bulat Gaifullin", @@ -442,7 +412,10 @@ def prepare_static_build_linux(self): maintainer='Oleg Hoefling', maintainer_email='oleg.hoefling@gmail.com', url='https://github.com/mehcode/python-xmlsec', - project_urls={'Documentation': 'https://xmlsec.readthedocs.io', 'Source': 'https://github.com/mehcode/python-xmlsec',}, + project_urls={ + 'Documentation': 'https://xmlsec.readthedocs.io', + 'Source': 'https://github.com/mehcode/python-xmlsec', + }, license='MIT', keywords=['xmlsec'], classifiers=[ @@ -452,12 +425,12 @@ def prepare_static_build_linux(self): 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: C', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From c147027772c61761c56c119f0b4296fade9a3380 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sun, 25 Apr 2021 20:10:07 +0200 Subject: [PATCH 019/211] drop set-env usage in gh actions Signed-off-by: oleg.hoefling --- .github/workflows/macosx.yml | 20 ++++++++++---------- .github/workflows/manylinux2010.yml | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 01302083..70e00291 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -17,28 +17,28 @@ jobs: pip install --upgrade pip setuptools wheel brew install libxml2 libxmlsec1 pkg-config - name: Build macosx_x86_64 wheel + env: + CC: clang + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" run: | python setup.py bdist_wheel + rm -rf build/ - name: Set environment variables shell: bash run: | - echo ::set-env name=PKGVER::$(python setup.py --version) - echo ::set-env name=LLVM_PROFILE_FILE::"pyxmlsec-%p.profraw" + echo "PKGVER=$(python setup.py --version)" >> $GITHUB_ENV + echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies - env: - CC: clang - CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" run: | - rm -rf build/ pip install coverage --upgrade -r requirements-test.txt pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV - name: Run tests run: | coverage run -m pytest -v --color=yes - name: Report coverage to codecov run: | - LIBFILE=$(python -c "import xmlsec; print(xmlsec.__file__)") - /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse pyxmlsec-*.profraw -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${LIBFILE} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 2190378b..046cb855 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -15,7 +15,7 @@ jobs: - name: Set environment variables shell: bash run: | - echo ::set-env name=PKGVER::$(/opt/python/${{ matrix.python-abi }}/bin/python setup.py --version) + echo "PKGVER=$(/opt/python/${{ matrix.python-abi }}/bin/python setup.py --version)" >> $GITHUB_ENV - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true @@ -24,10 +24,10 @@ jobs: - name: Label manylinux2010_x86_64 wheel run: | ls -la dist/ - auditwheel show dist/xmlsec-${PKGVER}-${{ matrix.python-abi }}-linux_x86_64.whl - auditwheel repair dist/xmlsec-${PKGVER}-${{ matrix.python-abi }}-linux_x86_64.whl + auditwheel show dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl + auditwheel repair dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl ls -l wheelhouse/ - auditwheel show wheelhouse/xmlsec-${PKGVER}-${{ matrix.python-abi }}-manylinux2010_x86_64.whl + auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-manylinux2010_x86_64.whl - name: Install test dependencies run: | /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade -r requirements-test.txt From ce45868701220368f156114978c784cc5200d80c Mon Sep 17 00:00:00 2001 From: Roman513 Date: Fri, 23 Apr 2021 23:45:17 +0300 Subject: [PATCH 020/211] Fix xmlsec version comparison macro Use 8 instead of 4 bits for all xmlsec version fields to prevent overflow starting from version number 1.2.32 --- src/constants.c | 9 ++++++--- src/main.c | 2 +- src/platform.h | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/constants.c b/src/constants.c index 68f086e9..b4cdedf4 100644 --- a/src/constants.c +++ b/src/constants.c @@ -452,7 +452,8 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif -#if XMLSEC_VERSION_HEX > 306 +#if XMLSEC_VERSION_HEX > 0x10212 + // from version 1.2.19 PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") @@ -502,7 +503,8 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDsaSha1, "DSA_SHA1"); #endif -#if XMLSEC_VERSION_HEX > 306 +#if XMLSEC_VERSION_HEX > 0x10212 + // from version 1.2.19 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformEcdsaSha1, "ECDSA_SHA1"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformEcdsaSha224, "ECDSA_SHA224"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformEcdsaSha256, "ECDSA_SHA256"); @@ -543,7 +545,8 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformSha384, "SHA384"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformSha512, "SHA512"); -#if XMLSEC_VERSION_HEX > 315 +#if XMLSEC_VERSION_HEX > 0x1021B + // from version 1.2.28 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes128Gcm, "AES128_GCM"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes192Gcm, "AES192_GCM"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformAes256Gcm, "AES256_GCM"); diff --git a/src/main.c b/src/main.c index 59989529..c097d7f5 100644 --- a/src/main.c +++ b/src/main.c @@ -27,7 +27,7 @@ static int free_mode = _PYXMLSEC_FREE_NONE; #ifndef XMLSEC_NO_CRYPTO_DYNAMIC_LOADING static const xmlChar* PyXmlSec_GetCryptoLibName() { -#if XMLSEC_VERSION_HEX > 308 +#if XMLSEC_VERSION_HEX > 0x10214 // xmlSecGetDefaultCrypto was introduced in version 1.2.21 const xmlChar* cryptoLib = xmlSecGetDefaultCrypto(); #else diff --git a/src/platform.h b/src/platform.h index 795062f2..01312a11 100644 --- a/src/platform.h +++ b/src/platform.h @@ -19,11 +19,11 @@ #include #endif /* MS_WIN32 */ -#define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 8) | (XMLSEC_VERSION_MINOR << 4) | (XMLSEC_VERSION_SUBMINOR)) +#define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 16) | (XMLSEC_VERSION_MINOR << 8) | (XMLSEC_VERSION_SUBMINOR)) // XKMS support was removed in version 1.2.21 // https://mail.gnome.org/archives/commits-list/2015-February/msg10555.html -#if XMLSEC_VERSION_HEX > 0x134 +#if XMLSEC_VERSION_HEX > 0x10214 #define XMLSEC_NO_XKMS 1 #endif From 7e76c45f152a8ef7a6e4b4937196a0ee9c2ed748 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Wed, 26 May 2021 13:03:57 +0200 Subject: [PATCH 021/211] add XMLSEC_NO_XSLT check Signed-off-by: oleg.hoefling --- src/constants.c | 4 +++- tests/test_ds.py | 2 ++ tests/test_templates.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/constants.c b/src/constants.c index b4cdedf4..93c6e434 100644 --- a/src/constants.c +++ b/src/constants.c @@ -485,7 +485,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformXPath, "XPATH"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformXPath2, "XPATH2"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformXPointer, "XPOINTER"); - PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformXslt, "XSLT"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRemoveXmlTagsC14N, "REMOVE_XML_TAGS_C14N"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformVisa3DHack, "VISA3D_HACK"); @@ -502,6 +501,9 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDsaSha1, "DSA_SHA1"); #endif +#ifndef XMLSEC_NO_XSLT + PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformXslt, "XSLT"); +#endif #if XMLSEC_VERSION_HEX > 0x10212 // from version 1.2.19 diff --git a/tests/test_ds.py b/tests/test_ds.py index cf49ee71..98b6424b 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -1,3 +1,4 @@ +import unittest import xmlsec from tests import base @@ -194,6 +195,7 @@ def test_sign_binary_no_key(self): with self.assertRaisesRegex(xmlsec.Error, 'Sign key is not specified.'): ctx.sign_binary(bytes=b'', transform=consts.TransformRsaSha1) + @unittest.skipIf(not hasattr(consts, 'TransformXslt'), reason='XSLT transformations not enabled') def test_sign_binary_invalid_signature_method(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) diff --git a/tests/test_templates.py b/tests/test_templates.py index 947e0bbc..9475c5e4 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,3 +1,4 @@ +import unittest from lxml import etree import xmlsec @@ -195,6 +196,7 @@ def test_encrypted_data_ensure_key_info_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.encrypted_data_ensure_key_info('') + @unittest.skipIf(not hasattr(consts, 'TransformXslt'), reason='XSLT transformations not enabled') def test_transform_add_c14n_inclusive_namespaces(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) From 609fac746c34dfcf34cf3c5e9d3f4ddf2c8c8dde Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Wed, 26 May 2021 13:13:39 +0200 Subject: [PATCH 022/211] drop manylinux wheel build for python 3.5 Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 046cb855..68211946 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -6,7 +6,7 @@ jobs: container: quay.io/pypa/manylinux2010_x86_64 strategy: matrix: - python-abi: [cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] steps: - uses: actions/checkout@v1 - name: Install build dependencies From db81f9e32eced602d640965ae6f17ac87e6f63e0 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Wed, 26 May 2021 14:22:33 +0200 Subject: [PATCH 023/211] prohibit static builds with libxml2-2.9.12, see https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 2 ++ .github/workflows/sdist.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 68211946..52e02ba2 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -19,6 +19,8 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true + # disable libxml2-2.9.12 because of https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 + PYXMLSEC_LIBXML2_VERSION: 2.9.10 run: | /opt/python/${{ matrix.python-abi }}/bin/python setup.py bdist_wheel - name: Label manylinux2010_x86_64 wheel diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 5b5dac96..d6cf6c9d 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -18,6 +18,8 @@ jobs: - name: Install test dependencies env: PYXMLSEC_STATIC_DEPS: true + # disable libxml2-2.9.12 because of https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 + PYXMLSEC_LIBXML2_VERSION: 2.9.10 run: | pip install --upgrade -r requirements-test.txt pip install dist/xmlsec-$(python setup.py --version).tar.gz From cd7ca859921e1ae66c6bf9fb3f6bc3b78f2146af Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Wed, 26 May 2021 14:28:24 +0200 Subject: [PATCH 024/211] adjust to PEP 600 changes Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 52e02ba2..e2b1f449 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -29,7 +29,7 @@ jobs: auditwheel show dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl auditwheel repair dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl ls -l wheelhouse/ - auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-manylinux2010_x86_64.whl + auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-manylinux_2_12_x86_64.manylinux2010_x86_64.whl - name: Install test dependencies run: | /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade -r requirements-test.txt From 95ed6fd12cc13d563d79f34bbe339e2d21fe88f8 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Wed, 26 May 2021 14:58:53 +0200 Subject: [PATCH 025/211] add linuxbrew build Signed-off-by: oleg.hoefling --- .github/workflows/linuxbrew.yml | 33 +++++++++++++++++++++++++++++++++ README.rst | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 .github/workflows/linuxbrew.yml diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml new file mode 100644 index 00000000..6502b830 --- /dev/null +++ b/.github/workflows/linuxbrew.yml @@ -0,0 +1,33 @@ +name: linuxbrew +on: [push, pull_request] +jobs: + linuxbrew: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install build dependencies + run: | + sudo apt install -y build-essential procps curl file git + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + test -d ~/.linuxbrew && eval $(~/.linuxbrew/bin/brew shellenv) + test -r ~/.bash_profile && echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.bash_profile + echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile + brew update + brew install python gcc libxml2 libxmlsec1 pkg-config + pip3 install --upgrade setuptools wheel + ln -s $(brew --prefix)/bin/gcc-11 $(brew --prefix)/bin/gcc-5 + - name: Build linux_x86_64 wheel + run: | + python3 setup.py bdist_wheel + rm -rf build/ + - name: Install test dependencies + run: | + pip3 install --upgrade -r requirements-test.txt + pip3 install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + - name: Run tests + run: | + pytest -v --color=yes diff --git a/README.rst b/README.rst index f01d467d..22235af4 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,8 @@ python-xmlsec :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22manylinux2010%22 .. image:: https://github.com/mehcode/python-xmlsec/workflows/MacOS/badge.svg :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22MacOS%22 +.. image:: https://github.com/mehcode/python-xmlsec/workflows/linuxbrew/badge.svg + :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22linuxbrew%22 .. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg :target: https://codecov.io/gh/mehcode/python-xmlsec .. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs From 6a1374d2a09d5e969c3379a95c2a234c4faff7b5 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:14:46 +0200 Subject: [PATCH 026/211] replace PyString_FromString usages with PyUnicode_FromString Signed-off-by: oleg.hoefling --- src/constants.c | 22 +++++++++++----------- src/keys.c | 2 +- src/platform.h | 2 -- src/template.c | 2 +- src/utils.c | 2 +- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/constants.c b/src/constants.c index 93c6e434..66aa88c7 100644 --- a/src/constants.c +++ b/src/constants.c @@ -27,7 +27,7 @@ static PyObject* PyXmlSec_Transform__str__(PyObject* self) { else snprintf(buf, sizeof(buf), "%s, None", transform->id->name); - return PyString_FromString(buf); + return PyUnicode_FromString(buf); } // __repr__ method @@ -38,18 +38,18 @@ static PyObject* PyXmlSec_Transform__repr__(PyObject* self) { snprintf(buf, sizeof(buf), "__Transform('%s', '%s', %d)", transform->id->name, transform->id->href, transform->id->usage); else snprintf(buf, sizeof(buf), "__Transform('%s', None, %d)", transform->id->name, transform->id->usage); - return PyString_FromString(buf); + return PyUnicode_FromString(buf); } static const char PyXmlSec_TransformNameGet__doc__[] = "The transform's name."; static PyObject* PyXmlSec_TransformNameGet(PyXmlSec_Transform* self, void* closure) { - return PyString_FromString((const char*)self->id->name); + return PyUnicode_FromString((const char*)self->id->name); } static const char PyXmlSec_TransformHrefGet__doc__[] = "The transform's identification string (href)."; static PyObject* PyXmlSec_TransformHrefGet(PyXmlSec_Transform* self, void* closure) { if (self->id->href != NULL) - return PyString_FromString((const char*)self->id->href); + return PyUnicode_FromString((const char*)self->id->href); Py_RETURN_NONE; } @@ -149,7 +149,7 @@ static PyObject* PyXmlSec_KeyData__str__(PyObject* self) { snprintf(buf, sizeof(buf), "%s, %s", keydata->id->name, keydata->id->href); else snprintf(buf, sizeof(buf), "%s, None", keydata->id->name); - return PyString_FromString(buf); + return PyUnicode_FromString(buf); } // __repr__ method @@ -160,18 +160,18 @@ static PyObject* PyXmlSec_KeyData__repr__(PyObject* self) { snprintf(buf, sizeof(buf), "__KeyData('%s', '%s')", keydata->id->name, keydata->id->href); else snprintf(buf, sizeof(buf), "__KeyData('%s', None)", keydata->id->name); - return PyString_FromString(buf); + return PyUnicode_FromString(buf); } static const char PyXmlSec_KeyDataNameGet__doc__[] = "The key data's name."; static PyObject* PyXmlSec_KeyDataNameGet(PyXmlSec_KeyData* self, void* closure) { - return PyString_FromString((const char*)self->id->name); + return PyUnicode_FromString((const char*)self->id->name); } static const char PyXmlSec_KeyDataHrefGet__doc__[] = "The key data's identification string (href)."; static PyObject* PyXmlSec_KeyDataHrefGet(PyXmlSec_KeyData* self, void* closure) { if (self->id->href != NULL) - return PyString_FromString((const char*)self->id->href); + return PyUnicode_FromString((const char*)self->id->href); Py_RETURN_NONE; } @@ -308,7 +308,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #define PYXMLSEC_ADD_NS_CONSTANT(name, lname) \ - tmp = PyString_FromString((const char*)(JOIN(xmlSec, name))); \ + tmp = PyUnicode_FromString((const char*)(JOIN(xmlSec, name))); \ PYXMLSEC_ADD_CONSTANT(nsCls, name, lname); // namespaces @@ -334,7 +334,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #define PYXMLSEC_ADD_ENC_CONSTANT(name, lname) \ - tmp = PyString_FromString((const char*)(JOIN(xmlSec, name))); \ + tmp = PyUnicode_FromString((const char*)(JOIN(xmlSec, name))); \ PYXMLSEC_ADD_CONSTANT(encryptionTypeCls, name, lname); // encryption type @@ -349,7 +349,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #define PYXMLSEC_ADD_NODE_CONSTANT(name, lname) \ - tmp = PyString_FromString((const char*)(JOIN(xmlSec, name))); \ + tmp = PyUnicode_FromString((const char*)(JOIN(xmlSec, name))); \ PYXMLSEC_ADD_CONSTANT(nodeCls, name, lname); // node diff --git a/src/keys.c b/src/keys.c index 357cc9c7..83419623 100644 --- a/src/keys.c +++ b/src/keys.c @@ -436,7 +436,7 @@ static PyObject* PyXmlSec_KeyNameGet(PyObject* self, void* closure) { } cname = (const char*)xmlSecKeyGetName(key->handle); if (cname != NULL) { - return PyString_FromString(cname); + return PyUnicode_FromString(cname); } Py_RETURN_NONE; } diff --git a/src/platform.h b/src/platform.h index 01312a11..a45b8b82 100644 --- a/src/platform.h +++ b/src/platform.h @@ -40,8 +40,6 @@ typedef int Py_ssize_t; #define PyString_Check PyUnicode_Check #define PyString_FromStringAndSize PyUnicode_FromStringAndSize -#define PyString_FromString PyUnicode_FromString - #define PyString_AsString PyUnicode_AsUTF8 #define PyString_AsUtf8AndSize PyUnicode_AsUTF8AndSize diff --git a/src/template.c b/src/template.c index 7d043606..9d19aaf3 100644 --- a/src/template.c +++ b/src/template.c @@ -766,7 +766,7 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, goto ON_FAIL; } if (PyList_Check(prefixes) || PyTuple_Check(prefixes)) { - sep = PyString_FromString(" "); + sep = PyUnicode_FromString(" "); prefixes = PyObject_CallMethod(sep, "join", "O", prefixes); Py_DECREF(sep); } else if (PyString_Check(prefixes)) { diff --git a/src/utils.c b/src/utils.c index ea6867f3..5f716128 100644 --- a/src/utils.c +++ b/src/utils.c @@ -32,7 +32,7 @@ PyObject* PyXmlSec_GetFilePathOrContent(PyObject* file, int* is_content) { } int PyXmlSec_SetStringAttr(PyObject* obj, const char* name, const char* value) { - PyObject* tmp = PyString_FromString(value); + PyObject* tmp = PyUnicode_FromString(value); int r; if (tmp == NULL) { From 908edee62f02a30ea9a780caa980d01a8f837642 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:16:21 +0200 Subject: [PATCH 027/211] replace PyString_Check usages with PyUnicode_Check Signed-off-by: oleg.hoefling --- src/platform.h | 2 -- src/template.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/platform.h b/src/platform.h index a45b8b82..dea74417 100644 --- a/src/platform.h +++ b/src/platform.h @@ -37,7 +37,6 @@ typedef int Py_ssize_t; #if PY_MAJOR_VERSION >= 3 #define PY3K 1 -#define PyString_Check PyUnicode_Check #define PyString_FromStringAndSize PyUnicode_FromStringAndSize #define PyString_AsString PyUnicode_AsUTF8 @@ -48,7 +47,6 @@ typedef int Py_ssize_t; #define PyString_FSConverter PyUnicode_FSConverter #else // PY3K -#define PyBytes_Check PyString_Check #define PyBytes_FromStringAndSize PyString_FromStringAndSize #define PyBytes_AsString PyString_AsString diff --git a/src/template.c b/src/template.c index 9d19aaf3..2fb87369 100644 --- a/src/template.c +++ b/src/template.c @@ -769,7 +769,7 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, sep = PyUnicode_FromString(" "); prefixes = PyObject_CallMethod(sep, "join", "O", prefixes); Py_DECREF(sep); - } else if (PyString_Check(prefixes)) { + } else if (PyUnicode_Check(prefixes)) { Py_INCREF(prefixes); } else { PyErr_SetString(PyExc_TypeError, "expected instance of str or list of str"); From 8977ff3232b54d56b25404e7f5ebd83cf6caf6ed Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:19:00 +0200 Subject: [PATCH 028/211] drop unused PyString_FromStringAndSize and PyString_AsUtf8AndSize Signed-off-by: oleg.hoefling --- src/platform.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/platform.h b/src/platform.h index dea74417..0062a930 100644 --- a/src/platform.h +++ b/src/platform.h @@ -37,26 +37,17 @@ typedef int Py_ssize_t; #if PY_MAJOR_VERSION >= 3 #define PY3K 1 -#define PyString_FromStringAndSize PyUnicode_FromStringAndSize #define PyString_AsString PyUnicode_AsUTF8 -#define PyString_AsUtf8AndSize PyUnicode_AsUTF8AndSize #define PyCreateDummyObject PyModule_New #define PyString_FSConverter PyUnicode_FSConverter #else // PY3K -#define PyBytes_FromStringAndSize PyString_FromStringAndSize - #define PyBytes_AsString PyString_AsString #define PyBytes_AsStringAndSize PyString_AsStringAndSize -static inline char* PyString_AsUtf8AndSize(PyObject *obj, Py_ssize_t* length) { - char* buffer = NULL; - return (PyString_AsStringAndSize(obj, &buffer, length) < 0) ? (char*)(0) : buffer; -} - static inline PyObject* PyCreateDummyObject(const char* name) { PyObject* tmp = Py_InitModule(name, NULL); Py_INCREF(tmp); From c1e269abdc36cf391ebdea45042fbf654e5fe92e Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:21:03 +0200 Subject: [PATCH 029/211] replace PyString_AsString usages with PyUnicode_AsUTF8 Signed-off-by: oleg.hoefling --- src/keys.c | 2 +- src/platform.h | 3 --- src/template.c | 2 +- src/tree.c | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/keys.c b/src/keys.c index 83419623..9e65be8f 100644 --- a/src/keys.c +++ b/src/keys.c @@ -460,7 +460,7 @@ static int PyXmlSec_KeyNameSet(PyObject* self, PyObject* value, void* closure) { return 0; } - name = PyString_AsString(value); + name = PyUnicode_AsUTF8(value); if (name == NULL) return -1; if (xmlSecKeySetName(key->handle, XSTR(name)) < 0) { diff --git a/src/platform.h b/src/platform.h index 0062a930..fe1c6f9b 100644 --- a/src/platform.h +++ b/src/platform.h @@ -38,14 +38,11 @@ typedef int Py_ssize_t; #if PY_MAJOR_VERSION >= 3 #define PY3K 1 -#define PyString_AsString PyUnicode_AsUTF8 - #define PyCreateDummyObject PyModule_New #define PyString_FSConverter PyUnicode_FSConverter #else // PY3K -#define PyBytes_AsString PyString_AsString #define PyBytes_AsStringAndSize PyString_AsStringAndSize static inline PyObject* PyCreateDummyObject(const char* name) { diff --git a/src/template.c b/src/template.c index 2fb87369..0d35832b 100644 --- a/src/template.c +++ b/src/template.c @@ -781,7 +781,7 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, } - c_prefixes = PyString_AsString(prefixes); + c_prefixes = PyUnicode_AsUTF8(prefixes); Py_BEGIN_ALLOW_THREADS; res = xmlSecTmplTransformAddC14NInclNamespaces(node->_c_node, XSTR(c_prefixes)); Py_END_ALLOW_THREADS; diff --git a/src/tree.c b/src/tree.c index 76037d3b..df8821c3 100644 --- a/src/tree.c +++ b/src/tree.c @@ -182,7 +182,7 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k tmp = PyObject_GetItem(ids, key); Py_DECREF(key); if (tmp == NULL) goto ON_FAIL; - list[i] = XSTR(PyString_AsString(tmp)); + list[i] = XSTR(PyUnicode_AsUTF8(tmp)); Py_DECREF(tmp); if (list[i] == NULL) goto ON_FAIL; } From d29be044a03aec424ecc0602a0ab8d79c7311d94 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:22:35 +0200 Subject: [PATCH 030/211] replace PyCreateDummyObject usages with PyModule_New Signed-off-by: oleg.hoefling --- src/constants.c | 2 +- src/platform.h | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/constants.c b/src/constants.c index 66aa88c7..938d5834 100644 --- a/src/constants.c +++ b/src/constants.c @@ -292,7 +292,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #undef PYXMLSEC_ADD_INT_CONSTANT #define PYXMLSEC_DECLARE_NAMESPACE(var, name) \ - if (!(var = PyCreateDummyObject(name))) goto ON_FAIL; \ + if (!(var = PyModule_New(name))) goto ON_FAIL; \ if (PyModule_AddObject(package, name, var) < 0) goto ON_FAIL; \ Py_INCREF(var); // add object steels reference diff --git a/src/platform.h b/src/platform.h index fe1c6f9b..25c24d19 100644 --- a/src/platform.h +++ b/src/platform.h @@ -38,19 +38,11 @@ typedef int Py_ssize_t; #if PY_MAJOR_VERSION >= 3 #define PY3K 1 -#define PyCreateDummyObject PyModule_New - #define PyString_FSConverter PyUnicode_FSConverter #else // PY3K #define PyBytes_AsStringAndSize PyString_AsStringAndSize -static inline PyObject* PyCreateDummyObject(const char* name) { - PyObject* tmp = Py_InitModule(name, NULL); - Py_INCREF(tmp); - return tmp; -} - static inline int PyString_FSConverter(PyObject* o, PyObject** p) { if (o == NULL) { return 0; From cfe61ea2c4040a5ce11d003ef894412eddc7be89 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:26:07 +0200 Subject: [PATCH 031/211] replace PyString_FSConverter usages with PyUnicode_FSConverter Signed-off-by: oleg.hoefling --- src/keys.c | 4 ++-- src/platform.h | 12 ------------ src/utils.c | 2 +- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/keys.c b/src/keys.c index 9e65be8f..8e065ea0 100644 --- a/src/keys.c +++ b/src/keys.c @@ -249,7 +249,7 @@ static PyObject* PyXmlSec_KeyFromBinaryFile(PyObject* self, PyObject* args, PyOb PYXMLSEC_DEBUG("load symmetric key - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!O&:from_binary_file", kwlist, - PyXmlSec_KeyDataType, &keydata, PyString_FSConverter, &filepath)) + PyXmlSec_KeyDataType, &keydata, PyUnicode_FSConverter, &filepath)) { goto ON_FAIL; } @@ -698,7 +698,7 @@ static PyObject* PyXmlSec_KeysManagerLoadCert(PyObject* self, PyObject* args, Py PYXMLSEC_DEBUGF("%p: load cert - start", self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&II:load_cert", kwlist, - PyString_FSConverter, &filepath, &format, &type)) { + PyUnicode_FSConverter, &filepath, &format, &type)) { goto ON_FAIL; } diff --git a/src/platform.h b/src/platform.h index 25c24d19..35705f2d 100644 --- a/src/platform.h +++ b/src/platform.h @@ -37,22 +37,10 @@ typedef int Py_ssize_t; #if PY_MAJOR_VERSION >= 3 #define PY3K 1 - -#define PyString_FSConverter PyUnicode_FSConverter #else // PY3K #define PyBytes_AsStringAndSize PyString_AsStringAndSize -static inline int PyString_FSConverter(PyObject* o, PyObject** p) { - if (o == NULL) { - return 0; - } - - Py_INCREF(o); - *p = o; - return 1; -} - #endif // PYTHON3 static inline char* PyBytes_AsStringAndSize2(PyObject *obj, Py_ssize_t* length) { diff --git a/src/utils.c b/src/utils.c index 5f716128..cdcb182b 100644 --- a/src/utils.c +++ b/src/utils.c @@ -25,7 +25,7 @@ PyObject* PyXmlSec_GetFilePathOrContent(PyObject* file, int* is_content) { return data; } *is_content = 0; - if (!PyString_FSConverter(file, &tmp)) { + if (!PyUnicode_FSConverter(file, &tmp)) { return NULL; } return tmp; From 5055be674a2de83094900591935a096173d74e1d Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 17:48:11 +0200 Subject: [PATCH 032/211] remove python 2 code bits Signed-off-by: oleg.hoefling --- src/constants.c | 7 ------- src/main.c | 51 ------------------------------------------------- src/platform.h | 8 -------- src/template.c | 7 ------- src/tree.c | 7 ------- 5 files changed, 80 deletions(-) diff --git a/src/constants.c b/src/constants.c index 938d5834..8c17e389 100644 --- a/src/constants.c +++ b/src/constants.c @@ -245,7 +245,6 @@ static PyObject* PyXmlSec_KeyDataNew(xmlSecKeyDataId id) { return (PyObject*)keydata; } -#ifdef PY3K static PyModuleDef PyXmlSec_ConstantsModule = { PyModuleDef_HEAD_INIT, @@ -253,7 +252,6 @@ static PyModuleDef PyXmlSec_ConstantsModule = PYXMLSEC_CONSTANTS_DOC, -1, NULL, NULL, NULL, NULL, NULL }; -#endif // PY3K // initialize constants module and registers it base package int PyXmlSec_ConstantsModule_Init(PyObject* package) { @@ -267,12 +265,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PyObject* keyDataTypeCls = NULL; PyObject* tmp = NULL; -#ifdef PY3K constants = PyModule_Create(&PyXmlSec_ConstantsModule); -#else - constants = Py_InitModule3(STRINGIFY(MODULE_NAME) ".constants", NULL, PYXMLSEC_CONSTANTS_DOC); - Py_XINCREF(constants); -#endif if (!constants) return -1; diff --git a/src/main.c b/src/main.c index c097d7f5..a602982c 100644 --- a/src/main.c +++ b/src/main.c @@ -203,8 +203,6 @@ int PyXmlSec_EncModule_Init(PyObject* package); // templates management int PyXmlSec_TemplateModule_Init(PyObject* package); -#ifdef PY3K - static int PyXmlSec_PyClear(PyObject *self) { PyXmlSec_Free(free_mode); return 0; @@ -225,54 +223,12 @@ static PyModuleDef PyXmlSecModule = { #define PYENTRY_FUNC_NAME JOIN(PyInit_, MODULE_NAME) #define PY_MOD_RETURN(m) return m -#else // PY3K -#define PYENTRY_FUNC_NAME JOIN(init, MODULE_NAME) -#define PY_MOD_RETURN(m) return - -static void PyXmlSec_PyModuleGuard__del__(PyObject* self) -{ - PyXmlSec_Free(free_mode); - Py_TYPE(self)->tp_free(self); -} - -// we need guard to free resources on module unload -typedef struct { - PyObject_HEAD -} PyXmlSec_PyModuleGuard; - -static PyTypeObject PyXmlSec_PyModuleGuardType = { - PyVarObject_HEAD_INIT(NULL, 0) - STRINGIFY(MODULE_NAME) "__Guard", /* tp_name */ - sizeof(PyXmlSec_PyModuleGuard), /* tp_basicsize */ - 0, /* tp_itemsize */ - PyXmlSec_PyModuleGuard__del__, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ -}; -#endif // PY3K PyMODINIT_FUNC PYENTRY_FUNC_NAME(void) { PyObject *module = NULL; -#ifdef PY3K module = PyModule_Create(&PyXmlSecModule); -#else - module = Py_InitModule3(STRINGIFY(MODULE_NAME), PyXmlSec_MainMethods, MODULE_DOC); -#endif if (!module) { PY_MOD_RETURN(NULL); /* this really should never happen */ } @@ -294,13 +250,6 @@ PYENTRY_FUNC_NAME(void) if (PyXmlSec_EncModule_Init(module) < 0) goto ON_FAIL; if (PyXmlSec_TemplateModule_Init(module) < 0) goto ON_FAIL; -#ifndef PY3K - if (PyType_Ready(&PyXmlSec_PyModuleGuardType) < 0) goto ON_FAIL; - PYXMLSEC_DEBUGF("%p", &PyXmlSec_PyModuleGuardType); - // added guard to free resources on module unload, this should be called after last - if (PyModule_AddObject(module, "__guard", _PyObject_New(&PyXmlSec_PyModuleGuardType)) < 0) goto ON_FAIL; -#endif - PY_MOD_RETURN(module); ON_FAIL: PY_MOD_RETURN(NULL); diff --git a/src/platform.h b/src/platform.h index 35705f2d..35163e88 100644 --- a/src/platform.h +++ b/src/platform.h @@ -35,14 +35,6 @@ typedef int Py_ssize_t; #define PY_SSIZE_T_MIN INT_MIN #endif -#if PY_MAJOR_VERSION >= 3 -#define PY3K 1 -#else // PY3K - -#define PyBytes_AsStringAndSize PyString_AsStringAndSize - -#endif // PYTHON3 - static inline char* PyBytes_AsStringAndSize2(PyObject *obj, Py_ssize_t* length) { char* buffer = NULL; return ((PyBytes_AsStringAndSize(obj, &buffer, length) < 0) ? (char*)(0) : buffer); diff --git a/src/template.c b/src/template.c index 0d35832b..ae0eca34 100644 --- a/src/template.c +++ b/src/template.c @@ -918,7 +918,6 @@ static PyMethodDef PyXmlSec_TemplateMethods[] = { {NULL, NULL} /* sentinel */ }; -#ifdef PY3K static PyModuleDef PyXmlSec_TemplateModule = { PyModuleDef_HEAD_INIT, @@ -931,15 +930,9 @@ static PyModuleDef PyXmlSec_TemplateModule = NULL, /* m_clear */ NULL, /* m_free */ }; -#endif // PY3K int PyXmlSec_TemplateModule_Init(PyObject* package) { -#ifdef PY3K PyObject* template = PyModule_Create(&PyXmlSec_TemplateModule); -#else - PyObject* template = Py_InitModule3(STRINGIFY(MODULE_NAME) ".template", PyXmlSec_TemplateMethods, PYXMLSEC_TEMPLATES_DOC); - Py_XINCREF(template); -#endif if (!template) goto ON_FAIL; PYXMLSEC_DEBUGF("%p", template); diff --git a/src/tree.c b/src/tree.c index df8821c3..37cae785 100644 --- a/src/tree.c +++ b/src/tree.c @@ -230,7 +230,6 @@ static PyMethodDef PyXmlSec_TreeMethods[] = { {NULL, NULL} /* sentinel */ }; -#ifdef PY3K static PyModuleDef PyXmlSec_TreeModule = { PyModuleDef_HEAD_INIT, @@ -243,16 +242,10 @@ static PyModuleDef PyXmlSec_TreeModule = NULL, /* m_clear */ NULL, /* m_free */ }; -#endif // PY3K int PyXmlSec_TreeModule_Init(PyObject* package) { -#ifdef PY3K PyObject* tree = PyModule_Create(&PyXmlSec_TreeModule); -#else - PyObject* tree = Py_InitModule3(STRINGIFY(MODULE_NAME) ".tree", PyXmlSec_TreeMethods, PYXMLSEC_TREE_DOC); - Py_XINCREF(tree); -#endif if (!tree) goto ON_FAIL; From 268c5881007c400227283647aa7b90c99a28f6e4 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 22:00:58 +0200 Subject: [PATCH 033/211] work around xmlsec overwriting custom callback on init Signed-off-by: oleg.hoefling --- pyproject.toml | 9 +-------- src/exception.c | 10 +++++++--- src/exception.h | 2 ++ src/main.c | 5 +++++ tests/base.py | 12 ++++++------ tests/test_constants.py | 3 ++- tests/test_ds.py | 5 +++-- tests/test_enc.py | 2 +- tests/test_keys.py | 22 +++++++++++----------- tests/test_templates.py | 5 +++-- tests/test_xmlsec.py | 4 ++-- 11 files changed, 43 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 636c52c9..6c69db80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,14 +18,7 @@ exclude = ''' ''' [tool.isort] -force_alphabetical_sort_within_sections = true -recursive = true -line_length = 130 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -combine_as_imports = true +profile = 'black' known_first_party = ['xmlsec'] known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] diff --git a/src/exception.c b/src/exception.c index 2ca5ab57..176cf385 100644 --- a/src/exception.c +++ b/src/exception.c @@ -165,6 +165,12 @@ void PyXmlSecEnableDebugTrace(int v) { PyXmlSec_PrintErrorMessage = v; } +void PyXmlSec_InstallErrorCallback() { + if (PyXmlSec_LastErrorKey != 0) { + xmlSecErrorsSetCallback(PyXmlSec_ErrorCallback); + } +} + // initializes errors module int PyXmlSec_ExceptionsModule_Init(PyObject* package) { PyXmlSec_Error = NULL; @@ -185,9 +191,7 @@ int PyXmlSec_ExceptionsModule_Init(PyObject* package) { if (PyModule_AddObject(package, "VerificationError", PyXmlSec_VerificationError) < 0) goto ON_FAIL; PyXmlSec_LastErrorKey = PyThread_create_key(); - if (PyXmlSec_LastErrorKey != 0) { - xmlSecErrorsSetCallback(&PyXmlSec_ErrorCallback); - } + PyXmlSec_InstallErrorCallback(); return 0; diff --git a/src/exception.h b/src/exception.h index 9dea5ecb..687cd778 100644 --- a/src/exception.h +++ b/src/exception.h @@ -24,4 +24,6 @@ void PyXmlSec_ClearError(void); void PyXmlSecEnableDebugTrace(int); +void PyXmlSec_InstallErrorCallback(); + #endif //__PYXMLSEC_EXCEPTIONS_H__ diff --git a/src/main.c b/src/main.c index a602982c..c93b16d2 100644 --- a/src/main.c +++ b/src/main.c @@ -239,6 +239,11 @@ PYENTRY_FUNC_NAME(void) if (PyXmlSec_Init() < 0) goto ON_FAIL; + // xmlsec will install default callback in PyXmlSec_Init, + // overwriting any custom callbacks. + // We thus install our callback again now. + PyXmlSec_InstallErrorCallback(); + if (PyModule_AddStringConstant(module, "__version__", STRINGIFY(MODULE_VERSION)) < 0) goto ON_FAIL; if (PyXmlSec_InitLxmlModule() < 0) goto ON_FAIL; diff --git a/tests/base.py b/tests/base.py index cf659b61..e834f080 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,13 +1,13 @@ import gc import os import sys +import unittest from lxml import etree -import xmlsec -import unittest +import xmlsec -if sys.version_info < (3, ): +if sys.version_info < (3,): unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp @@ -21,6 +21,8 @@ def get_memory_usage(): return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + except ImportError: resource = None @@ -112,9 +114,7 @@ def assertXmlEqual(self, first, second, msg=None): self.fail('Tags do not match: %s and %s. %s' % (first.tag, second.tag, msg)) for name, value in first.attrib.items(): if second.attrib.get(name) != value: - self.fail( - 'Attributes do not match: %s=%r, %s=%r. %s' % (name, value, name, second.attrib.get(name), msg) - ) + self.fail('Attributes do not match: %s=%r, %s=%r. %s' % (name, value, name, second.attrib.get(name), msg)) for name in second.attrib.keys(): if name not in first.attrib: self.fail('x2 has an attribute x1 is missing: %s. %s' % (name, msg)) diff --git a/tests/test_constants.py b/tests/test_constants.py index 857a1cdd..689edce6 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,8 +1,9 @@ """Test constants from :mod:`xmlsec.constants` module.""" -import xmlsec from hypothesis import given, strategies +import xmlsec + def _constants(typename): return list( diff --git a/tests/test_ds.py b/tests/test_ds.py index 98b6424b..9417fedb 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -1,4 +1,5 @@ import unittest + import xmlsec from tests import base @@ -70,7 +71,7 @@ def test_sign_bad_args(self): def test_sign_fail(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) - with self.assertRaisesRegex(xmlsec.InternalError, 'failed to sign'): + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): ctx.sign(self.load_xml('sign1-in.xml')) def test_sign_case1(self): @@ -229,7 +230,7 @@ def test_verify_bad_args(self): def test_verify_fail(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) - with self.assertRaisesRegex(xmlsec.InternalError, 'failed to verify'): + with self.assertRaisesRegex(xmlsec.Error, 'failed to verify'): ctx.verify(self.load_xml('sign1-in.xml')) def test_verify_case_1(self): diff --git a/tests/test_enc.py b/tests/test_enc.py index 141e5753..7add848c 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -191,7 +191,7 @@ def test_encrypt_uri_bad_args(self): def test_encrypt_uri_fail(self): ctx = xmlsec.EncryptionContext() - with self.assertRaisesRegex(xmlsec.InternalError, 'failed to encrypt URI'): + with self.assertRaisesRegex(xmlsec.Error, 'failed to encrypt URI'): ctx.encrypt_uri(etree.Element('root'), '') def test_decrypt1(self): diff --git a/tests/test_keys.py b/tests/test_keys.py index 12b8224f..0d41abef 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -17,7 +17,7 @@ def test_key_from_memory_with_bad_args(self): xmlsec.Key.from_memory(1, format="") def test_key_from_memory_invalid_data(self): - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load key.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load key.*'): xmlsec.Key.from_memory(b'foo', format=consts.KeyDataFormatPem) def test_key_from_file(self): @@ -29,7 +29,7 @@ def test_key_from_file_with_bad_args(self): xmlsec.Key.from_file(1, format="") def test_key_from_invalid_file(self): - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): with tempfile.NamedTemporaryFile() as tmpfile: tmpfile.write(b'foo') xmlsec.Key.from_file(tmpfile.name, format=consts.KeyDataFormatPem) @@ -42,7 +42,7 @@ def test_key_from_fileobj(self): def test_key_from_invalid_fileobj(self): with tempfile.NamedTemporaryFile(delete=False) as tmpfile: tmpfile.write(b'foo') - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'), open(tmpfile.name) as fp: + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'), open(tmpfile.name) as fp: xmlsec.Key.from_file(fp, format=consts.KeyDataFormatPem) def test_generate(self): @@ -54,7 +54,7 @@ def test_generate_with_bad_args(self): xmlsec.Key.generate(klass="", size="", type="") def test_generate_invalid_size(self): - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot generate key.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot generate key.*'): xmlsec.Key.generate(klass=consts.KeyDataAes, size=0, type=consts.KeyDataTypeSession) def test_from_binary_file(self): @@ -66,7 +66,7 @@ def test_from_binary_file_with_bad_args(self): xmlsec.Key.from_binary_file(klass="", filename=1) def test_from_invalid_binary_file(self): - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): with tempfile.NamedTemporaryFile() as tmpfile: tmpfile.write(b'foo') xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=tmpfile.name) @@ -80,7 +80,7 @@ def test_from_binary_data_with_bad_args(self): xmlsec.Key.from_binary_data(klass="", data=1) def test_from_invalid_binary_data(self): - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot read key.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=b'') def test_load_cert_from_file(self): @@ -97,7 +97,7 @@ def test_load_cert_from_file_with_bad_args(self): def test_load_cert_from_invalid_file(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): with tempfile.NamedTemporaryFile() as tmpfile: tmpfile.write(b'foo') key.load_cert_from_file(tmpfile.name, format=consts.KeyDataFormatPem) @@ -119,7 +119,7 @@ def test_load_cert_from_invalid_fileobj(self): self.assertIsNotNone(key) with tempfile.NamedTemporaryFile(delete=False) as tmpfile: tmpfile.write(b'foo') - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'), open(tmpfile.name) as fp: + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'), open(tmpfile.name) as fp: key.load_cert_from_file(fp, format=consts.KeyDataFormatPem) def test_load_cert_from_memory(self): @@ -136,7 +136,7 @@ def test_load_cert_from_memory_with_bad_args(self): def test_load_cert_from_memory_invalid_data(self): key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): key.load_cert_from_memory(b'', format=consts.KeyDataFormatPem) def test_get_name(self): @@ -190,7 +190,7 @@ def test_load_cert(self): def test_load_cert_with_bad_args(self): mngr = xmlsec.KeysManager() mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): with tempfile.NamedTemporaryFile() as tmpfile: tmpfile.write(b'foo') mngr.load_cert(tmpfile.name, format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) @@ -215,7 +215,7 @@ def test_load_cert_from_memory_with_bad_args(self): def test_load_cert_from_memory_invalid_data(self): mngr = xmlsec.KeysManager() mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) - with self.assertRaisesRegex(xmlsec.InternalError, '.*cannot load cert.*'): + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): mngr.load_cert_from_memory(b'', format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) def test_load_invalid_key(self): diff --git a/tests/test_templates.py b/tests/test_templates.py index 9475c5e4..3bae7e55 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -1,4 +1,5 @@ import unittest + from lxml import etree import xmlsec @@ -36,7 +37,7 @@ def test_ensure_key_info(self): self.assertEqual("Id", ki.get("Id")) def test_ensure_key_info_fail(self): - with self.assertRaisesRegex(xmlsec.InternalError, 'cannot ensure key info.'): + with self.assertRaisesRegex(xmlsec.Error, 'cannot ensure key info.'): xmlsec.template.ensure_key_info(etree.fromstring(b''), id="Id") def test_ensure_key_info_bad_args(self): @@ -88,7 +89,7 @@ def test_add_reference_bad_args(self): xmlsec.template.add_reference(etree.Element('root'), '') def test_add_reference_fail(self): - with self.assertRaisesRegex(xmlsec.InternalError, 'cannot add reference.'): + with self.assertRaisesRegex(xmlsec.Error, 'cannot add reference.'): xmlsec.template.add_reference(etree.Element('root'), consts.TransformSha1) def test_add_transform_bad_args(self): diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 32fac69a..2d470a11 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -10,5 +10,5 @@ def test_reinitialize_module(self): tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. """ - xmlsec.shutdown() - xmlsec.init() + # xmlsec.shutdown() + # xmlsec.init() From 03cd2ce07ca481f47607410a92f95f7c7f5a9009 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 22:44:58 +0200 Subject: [PATCH 034/211] switch to furo theme Signed-off-by: oleg.hoefling --- .gitignore | 2 +- .readthedocs.yml => .readthedocs.yaml | 2 +- doc/source/conf.py | 3 +-- doc/source/requirements.txt | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) rename .readthedocs.yml => .readthedocs.yaml (91%) diff --git a/.gitignore b/.gitignore index 1ef359b3..669df482 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ !.travis* !.appveyor* !.git* -!.readthedocs.yml +!.readthedocs.yaml # Python /dist diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 91% rename from .readthedocs.yml rename to .readthedocs.yaml index 4d8647be..49129b19 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yaml @@ -4,7 +4,7 @@ sphinx: configuration: doc/source/conf.py python: - version: 3.7 + version: 3.9 install: - method: pip path: . diff --git a/doc/source/conf.py b/doc/source/conf.py index bb75403b..8e39a5b6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -4,7 +4,6 @@ import urllib.request import lxml - from docutils.nodes import reference from packaging.version import parse from sphinx.errors import ExtensionError @@ -35,7 +34,7 @@ pygments_style = 'sphinx' todo_include_todos = False -html_theme = 'nature' +html_theme = 'furo' html_static_path = [] htmlhelp_basename = 'python-xmlsecdoc' diff --git a/doc/source/requirements.txt b/doc/source/requirements.txt index 09ff0002..6b78694a 100644 --- a/doc/source/requirements.txt +++ b/doc/source/requirements.txt @@ -2,3 +2,4 @@ lxml>=3.8 importlib_metadata;python_version < '3.8' packaging Sphinx>=3 +furo>=2021.4.11b34 From 59d6db21ee539b54cb872ffe11621cf3a721484d Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 22:57:17 +0200 Subject: [PATCH 035/211] use python 3.8 to build docs as rtd doesn't support 3.9 yet Signed-off-by: oleg.hoefling --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 49129b19..f89031d3 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,7 +4,7 @@ sphinx: configuration: doc/source/conf.py python: - version: 3.9 + version: 3.8 install: - method: pip path: . From 94f0fef0a4ec78f16dcb1270deccc86f0274ef95 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 29 May 2021 00:13:38 +0200 Subject: [PATCH 036/211] reinstall custom error callback after xmlSecCryptoInit() Signed-off-by: oleg.hoefling --- src/main.c | 10 +++++----- tests/test_xmlsec.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main.c b/src/main.c index c93b16d2..eef975a9 100644 --- a/src/main.c +++ b/src/main.c @@ -87,6 +87,11 @@ static int PyXmlSec_Init(void) { PyXmlSec_Free(_PYXMLSEC_FREE_ALL); return -1; } + // xmlsec will install default callback in xmlSecCryptoInit, + // overwriting any custom callbacks. + // We thus reinstall our callback now. + PyXmlSec_InstallErrorCallback(); + free_mode = _PYXMLSEC_FREE_ALL; return 0; } @@ -239,11 +244,6 @@ PYENTRY_FUNC_NAME(void) if (PyXmlSec_Init() < 0) goto ON_FAIL; - // xmlsec will install default callback in PyXmlSec_Init, - // overwriting any custom callbacks. - // We thus install our callback again now. - PyXmlSec_InstallErrorCallback(); - if (PyModule_AddStringConstant(module, "__version__", STRINGIFY(MODULE_VERSION)) < 0) goto ON_FAIL; if (PyXmlSec_InitLxmlModule() < 0) goto ON_FAIL; diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 2d470a11..32fac69a 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -10,5 +10,5 @@ def test_reinitialize_module(self): tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. """ - # xmlsec.shutdown() - # xmlsec.init() + xmlsec.shutdown() + xmlsec.init() From df14fa420f1978e52573188d231c49dd3d3670c9 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Thu, 27 Aug 2020 10:18:50 +0200 Subject: [PATCH 037/211] replace private type stubs with lxml-stubs Signed-off-by: oleg.hoefling --- mypy.ini | 1 - requirements-test.txt | 1 + typeshed/lxml/__init__.pyi | 0 typeshed/lxml/etree.pyi | 6 ------ 4 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 typeshed/lxml/__init__.pyi delete mode 100644 typeshed/lxml/etree.pyi diff --git a/mypy.ini b/mypy.ini index ce190e35..f404c54c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,5 @@ [mypy] files = src -mypy_path = typeshed/ ignore_missing_imports = False warn_unused_configs = True disallow_subclassing_any = True diff --git a/requirements-test.txt b/requirements-test.txt index dde7de3d..a2a90bbc 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ -r requirements.txt pytest>=4.6.9 hypothesis +lxml-stubs diff --git a/typeshed/lxml/__init__.pyi b/typeshed/lxml/__init__.pyi deleted file mode 100644 index e69de29b..00000000 diff --git a/typeshed/lxml/etree.pyi b/typeshed/lxml/etree.pyi deleted file mode 100644 index 9420180f..00000000 --- a/typeshed/lxml/etree.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Any - -def __getattr__(name: str) -> Any: ... # incomplete - -class _Element: - def __getattr__(self, name: str) -> Any: ... # incomplete From b31246e947caa48e1756fc111393c76c13ad015b Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 29 May 2021 11:48:52 +0200 Subject: [PATCH 038/211] update xmlsec.constants stubs Signed-off-by: oleg.hoefling --- src/xmlsec/constants.pyi | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 8be27b23..97e00544 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -1,5 +1,5 @@ import sys -from typing import NamedTuple +from typing import NamedTuple, Optional if sys.version_info >= (3, 8): from typing import Final @@ -7,12 +7,12 @@ else: from typing_extensions import Final class __KeyData(NamedTuple): # __KeyData type - href: str name: str + href: Optional[str] class __Transform(NamedTuple): # __Transform type - href: str name: str + href: Optional[str] usage: int DSigNs: Final = 'http://www.w3.org/2000/09/xmldsig#' @@ -109,7 +109,9 @@ TransformInclC14N11WithComments: Final = __Transform( 'c14n11-with-comments', 'http://www.w3.org/2006/12/xml-c14n11#WithComments', 3 ) TransformInclC14NWithComments: Final = __Transform( - 'c14n-with-comments', 'http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments', 3 + 'c14n-with-comments', + 'http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments', + 3, ) TransformKWAes128: Final = __Transform('kw-aes128', 'http://www.w3.org/2001/04/xmlenc#kw-aes128', 16) TransformKWAes192: Final = __Transform('kw-aes192', 'http://www.w3.org/2001/04/xmlenc#kw-aes192', 16) From e271d8e5e957ee947c883d59ba6da0cfeedd8466 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Fri, 28 May 2021 18:09:25 +0200 Subject: [PATCH 039/211] support PEP 539 for Python>=3.7 Signed-off-by: oleg.hoefling --- src/exception.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/exception.c b/src/exception.c index 176cf385..06cbc61a 100644 --- a/src/exception.c +++ b/src/exception.c @@ -23,7 +23,11 @@ PyObject* PyXmlSec_Error; PyObject* PyXmlSec_InternalError; PyObject* PyXmlSec_VerificationError; +#if PY_MINOR_VERSION >= 7 +static Py_tss_t PyXmlSec_LastErrorKey; +#else static int PyXmlSec_LastErrorKey = 0; +#endif static int PyXmlSec_PrintErrorMessage = 0; @@ -71,16 +75,26 @@ static PyXmlSec_ErrorHolder* PyXmlSec_ExchangeLastError(PyXmlSec_ErrorHolder* e) PyXmlSec_ErrorHolder* v; int r; - if (PyXmlSec_LastErrorKey == 0) { + #if PY_MINOR_VERSION >= 7 + if (PyThread_tss_is_created(&PyXmlSec_LastErrorKey) != 0) { + #else + if (PyXmlSec_LastErrorKey != 0) { + #endif PYXMLSEC_DEBUG("WARNING: There is no error key."); PyXmlSec_ErrorHolderFree(e); return NULL; } // get_key_value and set_key_value are gil free + #if PY_MINOR_VERSION >= 7 + v = (PyXmlSec_ErrorHolder*)PyThread_tss_get(&PyXmlSec_LastErrorKey); + //PyThread_tss_delete(&PyXmlSec_LastErrorKey); + r = PyThread_tss_set(&PyXmlSec_LastErrorKey, (void*)e); + #else v = (PyXmlSec_ErrorHolder*)PyThread_get_key_value(PyXmlSec_LastErrorKey); PyThread_delete_key_value(PyXmlSec_LastErrorKey); r = PyThread_set_key_value(PyXmlSec_LastErrorKey, (void*)e); + #endif PYXMLSEC_DEBUGF("set_key_value returns %d", r); return v; } @@ -166,7 +180,11 @@ void PyXmlSecEnableDebugTrace(int v) { } void PyXmlSec_InstallErrorCallback() { + #if PY_MINOR_VERSION >= 7 + if (PyThread_tss_is_created(&PyXmlSec_LastErrorKey) != 0) { + #else if (PyXmlSec_LastErrorKey != 0) { + #endif xmlSecErrorsSetCallback(PyXmlSec_ErrorCallback); } } @@ -190,8 +208,14 @@ int PyXmlSec_ExceptionsModule_Init(PyObject* package) { if (PyModule_AddObject(package, "InternalError", PyXmlSec_InternalError) < 0) goto ON_FAIL; if (PyModule_AddObject(package, "VerificationError", PyXmlSec_VerificationError) < 0) goto ON_FAIL; + #if PY_MINOR_VERSION >= 7 + if (PyThread_tss_create(&PyXmlSec_LastErrorKey)) { + PyXmlSec_InstallErrorCallback(); + } + #else PyXmlSec_LastErrorKey = PyThread_create_key(); PyXmlSec_InstallErrorCallback(); + #endif return 0; From 461541eed4f86df68ad6df11fca4c16c0cd09074 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 29 May 2021 13:45:06 +0200 Subject: [PATCH 040/211] avoid incompatible pointer type warning Signed-off-by: oleg.hoefling --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index 8e065ea0..46dbf651 100644 --- a/src/keys.c +++ b/src/keys.c @@ -453,7 +453,7 @@ static int PyXmlSec_KeyNameSet(PyObject* self, PyObject* value, void* closure) { } if (value == NULL) { - if (xmlSecKeySetName(key->handle, value) < 0) { + if (xmlSecKeySetName(key->handle, NULL) < 0) { PyXmlSec_SetLastError("cannot delete name"); return -1; } From 4d108dd9e750791138edebf5ba786bf04d692e7c Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 5 Jun 2021 00:29:56 +0200 Subject: [PATCH 041/211] fix error key presence check Signed-off-by: oleg.hoefling --- src/exception.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/exception.c b/src/exception.c index 06cbc61a..ac0e44ee 100644 --- a/src/exception.c +++ b/src/exception.c @@ -76,9 +76,9 @@ static PyXmlSec_ErrorHolder* PyXmlSec_ExchangeLastError(PyXmlSec_ErrorHolder* e) int r; #if PY_MINOR_VERSION >= 7 - if (PyThread_tss_is_created(&PyXmlSec_LastErrorKey) != 0) { + if (PyThread_tss_is_created(&PyXmlSec_LastErrorKey) == 0) { #else - if (PyXmlSec_LastErrorKey != 0) { + if (PyXmlSec_LastErrorKey == 0) { #endif PYXMLSEC_DEBUG("WARNING: There is no error key."); PyXmlSec_ErrorHolderFree(e); @@ -209,7 +209,7 @@ int PyXmlSec_ExceptionsModule_Init(PyObject* package) { if (PyModule_AddObject(package, "VerificationError", PyXmlSec_VerificationError) < 0) goto ON_FAIL; #if PY_MINOR_VERSION >= 7 - if (PyThread_tss_create(&PyXmlSec_LastErrorKey)) { + if (PyThread_tss_create(&PyXmlSec_LastErrorKey) == 0) { PyXmlSec_InstallErrorCallback(); } #else From 8b2f5a53e091e6fda636c8e2605a46254d542dca Mon Sep 17 00:00:00 2001 From: visuman Date: Thu, 10 Jun 2021 16:12:43 +0530 Subject: [PATCH 042/211] fix warning related to use of dash seperated values Fixes https://github.com/mehcode/python-xmlsec/issues/182 Making changes in setup.cfg to fix the warning related to use of dash-seperated values --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index ada30e9b..c61fd65b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [metadata] -description-file = README.rst +description_file = README.rst [bdist_rpm] release = 1 -build-requires = pkg-config xmlsec1-devel libxml2-devel xmlsec1-openssl-devel +build_requires = pkg-config xmlsec1-devel libxml2-devel xmlsec1-openssl-devel group = Development/Libraries requires = xmlsec1 xmlsec1-openssl @@ -13,7 +13,7 @@ build-dir = doc/build all_files = 1 [upload_docs] -upload-dir = doc/build/html +upload_dir = doc/build/html [flake8] max-line-length = 130 From 2460f50a51ae3719cdab5dbeabc18b8ef8c71f52 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 12 Jun 2021 14:06:51 +0200 Subject: [PATCH 043/211] add XMLSEC_NO_MD5 check Signed-off-by: oleg.hoefling --- src/constants.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/constants.c b/src/constants.c index 8c17e389..34c81b29 100644 --- a/src/constants.c +++ b/src/constants.c @@ -507,7 +507,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformEcdsaSha512, "ECDSA_SHA512"); #endif +#ifndef XMLSEC_NO_MD5 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformHmacMd5, "HMAC_MD5"); +#endif + #ifndef XMLSEC_NO_RIPEMD160 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformHmacRipemd160, "HMAC_RIPEMD160"); #endif @@ -517,7 +520,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformHmacSha384, "HMAC_SHA384"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformHmacSha512, "HMAC_SHA512"); +#ifndef XMLSEC_NO_MD5 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRsaMd5, "RSA_MD5"); +#endif + #ifndef XMLSEC_NO_RIPEMD160 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRsaRipemd160, "RSA_RIPEMD160"); #endif @@ -529,7 +535,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRsaPkcs1, "RSA_PKCS1"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRsaOaep, "RSA_OAEP"); +#ifndef XMLSEC_NO_MD5 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformMd5, "MD5"); +#endif + #ifndef XMLSEC_NO_RIPEMD160 PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformRipemd160, "RIPEMD160"); #endif From 80c5fb3173d6eb07d6e065ba7f48cf6bf52c87b6 Mon Sep 17 00:00:00 2001 From: Tonye Jack Date: Mon, 26 Jul 2021 15:40:59 -0400 Subject: [PATCH 044/211] Added missing pkg-config dependency. This seems to be required when using python3-slim debian image. Resolves ```#10 182.5 Building wheels for collected packages: xmlsec #10 182.5 Building wheel for xmlsec (PEP 517): started #10 182.9 Building wheel for xmlsec (PEP 517): finished with status 'error' #10 182.9 ERROR: Command errored out with exit status 1: #10 182.9 command: /venv/bin/python3.6 /venv/lib/python3.6/site-packages/pip/_vendor/pep517/in_process/_in_process.py build_wheel /tmp/tmpr1_o4zak #10 182.9 cwd: /tmp/pip-install-amr3abkp/xmlsec_201c200071954684a9d29b2158df27bc #10 182.9 Complete output (20 lines): #10 182.9 /tmp/pip-build-env-hr3wr4uf/overlay/lib/python3.6/site-packages/setuptools/dist.py:700: UserWarning: Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead #10 182.9 % (opt, underscore_opt)) #10 182.9 /tmp/pip-build-env-hr3wr4uf/overlay/lib/python3.6/site-packages/setuptools/dist.py:700: UserWarning: Usage of dash-separated 'build-requires' will not be supported in future versions. Please use the underscore name 'build_requires' instead #10 182.9 % (opt, underscore_opt)) #10 182.9 /tmp/pip-build-env-hr3wr4uf/overlay/lib/python3.6/site-packages/setuptools/dist.py:700: UserWarning: Usage of dash-separated 'upload-dir' will not be supported in future versions. Please use the underscore name 'upload_dir' instead #10 182.9 % (opt, underscore_opt)) #10 182.9 running bdist_wheel #10 182.9 running build #10 182.9 running build_py #10 182.9 package init file 'src/xmlsec/__init__.py' not found (or not a regular file) #10 182.9 creating build #10 182.9 creating build/lib.linux-x86_64-3.6 #10 182.9 creating build/lib.linux-x86_64-3.6/xmlsec #10 182.9 copying src/xmlsec/py.typed -> build/lib.linux-x86_64-3.6/xmlsec #10 182.9 copying src/xmlsec/constants.pyi -> build/lib.linux-x86_64-3.6/xmlsec #10 182.9 copying src/xmlsec/template.pyi -> build/lib.linux-x86_64-3.6/xmlsec #10 182.9 copying src/xmlsec/__init__.pyi -> build/lib.linux-x86_64-3.6/xmlsec #10 182.9 copying src/xmlsec/tree.pyi -> build/lib.linux-x86_64-3.6/xmlsec #10 182.9 running build_ext #10 182.9 error: Unable to invoke pkg-config. #10 182.9 ---------------------------------------- #10 182.9 ERROR: Failed building wheel for xmlsec #10 182.9 ERROR: Could not build wheels for xmlsec which use PEP 517 and cannot be installed directly #10 182.9 Failed to build xmlsec``` --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 22235af4..1543f821 100644 --- a/README.rst +++ b/README.rst @@ -53,7 +53,7 @@ Linux (Debian) .. code-block:: bash - apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl Note: There is no required version of LibXML2 for Ubuntu Precise, From 8ed003c2887c46d47a84bbd31bcc05a3559479db Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 26 Jul 2021 17:31:55 +0100 Subject: [PATCH 045/211] add changelog to project_urls --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2838fe84..541eb029 100644 --- a/setup.py +++ b/setup.py @@ -415,6 +415,7 @@ def prepare_static_build_linux(self): project_urls={ 'Documentation': 'https://xmlsec.readthedocs.io', 'Source': 'https://github.com/mehcode/python-xmlsec', + 'Changelog': 'https://github.com/mehcode/python-xmlsec/releases', }, license='MIT', keywords=['xmlsec'], From 73711a8c70387fb3a9b42d6c6a35fe1ca9d1d0ef Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Mon, 26 Jul 2021 17:09:13 +0100 Subject: [PATCH 046/211] First cut of registering Python callbacks for xmlsec This implementation uses a global linked-list structure to hold onto the Python callbacks and registers a single C wrapper callback with xmlsec to dispatch xmlsec's calling back to the appropriate Python function. This was the simplest way I could think to emulate dynamically wrapping the Python calls in C code, because C doesn't have closures. A potential downside is that the state is all very global. Perhaps, however, that's no worse than the very global set of callbacks that xmlsec holds onto itself, so long as we don't have any strange threading stuff going on. In order to accommodate client code potentially interleaving registrations of the default callbacks in between their own callbacks, the linked list structure can have nodes with NULL function pointers in it, such that when iterating through Python callbacks we can suspend our iteration and delegate back to the defaults at the appropriate point. As such, we always actually have the Python binding's C callbacks registered with xmlsec, and every time we get asked to register the default callbacks, we note a NULL function pointer and then re-register our C callbacks. --- src/main.c | 242 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/src/main.c b/src/main.c index eef975a9..ec97ec3b 100644 --- a/src/main.c +++ b/src/main.c @@ -15,6 +15,7 @@ #include #include #include +#include #define _PYXMLSEC_FREE_NONE 0 #define _PYXMLSEC_FREE_XMLSEC 1 @@ -133,6 +134,229 @@ static PyObject* PyXmlSec_PyEnableDebugOutput(PyObject *self, PyObject* args, Py Py_RETURN_NONE; } +// NB: This whole thing assumes that the `xmlsec` callbacks are not re-entrant +// (i.e. that xmlsec won't come across a link in the reference it's processing +// and try to open that with these callbacks too). +typedef struct CbList { + PyObject* match_cb; + PyObject* open_cb; + PyObject* read_cb; + PyObject* close_cb; + struct CbList* next; +} CbList; + +static CbList* registered_callbacks = NULL; +static CbList* rcb_tail = NULL; + +static void RCBListAppend(CbList* cb_list_item) { + if (registered_callbacks == NULL) { + registered_callbacks = cb_list_item; + } else { + rcb_tail->next = cb_list_item; + } + rcb_tail = cb_list_item; +} + +static void RCBListClear() { + CbList* cb_list_item = registered_callbacks; + while (cb_list_item) { + Py_XDECREF(cb_list_item->match_cb); + Py_XDECREF(cb_list_item->open_cb); + Py_XDECREF(cb_list_item->read_cb); + Py_XDECREF(cb_list_item->close_cb); + CbList* next = cb_list_item->next; + free(cb_list_item); + cb_list_item = next; + } + registered_callbacks = NULL; + rcb_tail = NULL; +} + +// The currently executing set of Python callbacks: +static CbList* cur_cb_list_item = NULL; + +static int PyXmlSec_MatchCB(const char* filename) { + if (!cur_cb_list_item) { + cur_cb_list_item = registered_callbacks; + } + while (cur_cb_list_item && !cur_cb_list_item->match_cb) { + // Spool past any default callback placeholders executed since we were + // last called back: + cur_cb_list_item = cur_cb_list_item->next; + } + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* args = Py_BuildValue("(y)", filename); + while (cur_cb_list_item && cur_cb_list_item->match_cb) { + PyObject* result = PyObject_CallObject(cur_cb_list_item->match_cb, args); + if (result && PyObject_IsTrue(result)) { + Py_DECREF(result); + Py_DECREF(args); + PyGILState_Release(state); + return 1; + } + cur_cb_list_item = cur_cb_list_item->next; + } + // FIXME: why does having this decref of args cause a segfault?! + Py_DECREF(args); + PyGILState_Release(state); + return 0; +} + +static void* PyXmlSec_OpenCB(const char* filename) { + PyGILState_STATE state = PyGILState_Ensure(); + + // NB: Assumes the match callback left the current callback list item in the + // right place: + PyObject* args = Py_BuildValue("(y)", filename); + PyObject* result = PyObject_CallObject(cur_cb_list_item->open_cb, args); + Py_DECREF(args); + + PyGILState_Release(state); + return result; +} + +static int PyXmlSec_ReadCB(void* context, char* buffer, int len) { + PyGILState_STATE state = PyGILState_Ensure(); + + // NB: Assumes the match callback left the current callback list item in the + // right place: + PyObject* py_buffer = PyMemoryView_FromMemory(buffer, (Py_ssize_t) len, PyBUF_WRITE); + PyObject* args = Py_BuildValue("(OO)", context, py_buffer); + PyObject* py_bytes_read = PyObject_CallObject(cur_cb_list_item->read_cb, args); + Py_DECREF(args); + Py_DECREF(py_buffer); + int result; + if (py_bytes_read && PyLong_Check(py_bytes_read)) { + result = (int)PyLong_AsLong(py_bytes_read); + Py_DECREF(py_bytes_read); + } else { + result = EOF; + } + + PyGILState_Release(state); + return result; +} + +static int PyXmlSec_CloseCB(void* context) { + PyGILState_STATE state = PyGILState_Ensure(); + + PyObject* args = Py_BuildValue("(O)", context); + PyObject* result = PyObject_CallObject(cur_cb_list_item->close_cb, args); + Py_DECREF(args); + Py_DECREF(context); + Py_DECREF(result); + + PyGILState_Release(state); + // We reset `cur_cb_list_item` because we've finished processing the set of + // callbacks that was matched + cur_cb_list_item = NULL; + return 0; +} + +static char PyXmlSec_PyIOCleanupCallbacks__doc__[] = \ + "Unregister globally all sets of IO callbacks from xmlsec."; +static PyObject* PyXmlSec_PyIOCleanupCallbacks(PyObject *self) { + xmlSecIOCleanupCallbacks(); + // We always have callbacks registered to delegate to any Python callbacks + // we have registered within these bindings: + if (xmlSecIORegisterCallbacks( + PyXmlSec_MatchCB, PyXmlSec_OpenCB, PyXmlSec_ReadCB, + PyXmlSec_CloseCB) < 0) { + return NULL; + }; + RCBListClear(); + Py_RETURN_NONE; +} + +static char PyXmlSec_PyIORegisterDefaultCallbacks__doc__[] = \ + "Register globally xmlsec's own default set of IO callbacks."; +static PyObject* PyXmlSec_PyIORegisterDefaultCallbacks(PyObject *self) { + if (xmlSecIORegisterDefaultCallbacks() < 0) { + return NULL; + } + // We place a nulled item on the callback list to represent whenever the + // default callbacks are going to be invoked: + CbList* cb_list_item = malloc(sizeof(CbList)); + if (cb_list_item == NULL) { + return NULL; + } + cb_list_item->match_cb = NULL; + cb_list_item->open_cb = NULL; + cb_list_item->read_cb = NULL; + cb_list_item->close_cb = NULL; + cb_list_item->next = NULL; + RCBListAppend(cb_list_item); + // We need to make sure we can continue trying to match futher Python + // callbacks if the default callback doesn't match: + if (xmlSecIORegisterCallbacks( + PyXmlSec_MatchCB, PyXmlSec_OpenCB, PyXmlSec_ReadCB, + PyXmlSec_CloseCB) < 0) { + return NULL; + }; + Py_RETURN_NONE; +} + +static char PyXmlSec_PyIORegisterCallbacks__doc__[] = \ + "Register globally a custom set of IO callbacks with xmlsec.\n\n" + ":param callable input_match_callback: A callable that takes a filename `bytestring` and " + "returns a boolean as to whether the other callbacks in this set can handle that name.\n" + ":param callable input_open_callback: A callable that takes a filename and returns some " + "context object (e.g. a file object) that the remaining callables in this set will be passed " + "during handling.\n" + // FIXME: How do we handle failures in ^^ (e.g. can't find the file)? + ":param callable input_read_callback: A callable that that takes the context object from the " + "open callback and a buffer, and should fill the buffer with data (e.g. BytesIO.readinto()). " + "xmlsec will call this function several times until there is no more data returned.\n" + ":param callable input_close_callback: A callable that takes the context object from the " + "open callback and can do any resource cleanup necessary.\n" + ; +static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = { + "input_match_callback", + "input_open_callback", + "input_read_callback", + "input_close_callback", + NULL + }; + CbList* cb_list_item = malloc(sizeof(CbList)); + if (cb_list_item == NULL) { + return NULL; + } + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, "OOOO:register_callbacks", kwlist, + &cb_list_item->match_cb, &cb_list_item->open_cb, &cb_list_item->read_cb, + &cb_list_item->close_cb)) { + free(cb_list_item); + return NULL; + } + if (!PyCallable_Check(cb_list_item->match_cb)) { + PyErr_SetString(PyExc_TypeError, "input_match_callback must be a callable"); + return NULL; + } + if (!PyCallable_Check(cb_list_item->open_cb)) { + PyErr_SetString(PyExc_TypeError, "input_open_callback must be a callable"); + return NULL; + } + if (!PyCallable_Check(cb_list_item->read_cb)) { + PyErr_SetString(PyExc_TypeError, "input_read_callback must be a callable"); + return NULL; + } + if (!PyCallable_Check(cb_list_item->close_cb)) { + PyErr_SetString(PyExc_TypeError, "input_close_callback must be a callable"); + return NULL; + } + Py_INCREF(cb_list_item->match_cb); + Py_INCREF(cb_list_item->open_cb); + Py_INCREF(cb_list_item->read_cb); + Py_INCREF(cb_list_item->close_cb); + cb_list_item->next = NULL; + RCBListAppend(cb_list_item); + // NB: We don't need to register the callbacks with `xmlsec` here, because + // we've already registered our helper functions that will trawl through our + // list of callbacks. + Py_RETURN_NONE; +} + static char PyXmlSec_PyBase64DefaultLineSize__doc__[] = \ "base64_default_line_size(size = None)\n" "Configures the default maximum columns size for base64 encoding.\n\n" @@ -181,6 +405,24 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_VARARGS|METH_KEYWORDS, PyXmlSec_PyEnableDebugOutput__doc__ }, + { + "cleanup_callbacks", + (PyCFunction)PyXmlSec_PyIOCleanupCallbacks, + METH_NOARGS, + PyXmlSec_PyIOCleanupCallbacks__doc__ + }, + { + "register_default_callbacks", + (PyCFunction)PyXmlSec_PyIORegisterDefaultCallbacks, + METH_NOARGS, + PyXmlSec_PyIORegisterDefaultCallbacks__doc__ + }, + { + "register_callbacks", + (PyCFunction)PyXmlSec_PyIORegisterCallbacks, + METH_VARARGS|METH_KEYWORDS, + PyXmlSec_PyIORegisterCallbacks__doc__ + }, { "base64_default_line_size", (PyCFunction)PyXmlSec_PyBase64DefaultLineSize, From 09665dfa38ef10e13edf387a915ae273b71508a4 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Tue, 27 Jul 2021 10:18:10 +0100 Subject: [PATCH 047/211] Improve memory cleanup --- src/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main.c b/src/main.c index ec97ec3b..205174e6 100644 --- a/src/main.c +++ b/src/main.c @@ -194,9 +194,9 @@ static int PyXmlSec_MatchCB(const char* filename) { PyGILState_Release(state); return 1; } + Py_XDECREF(result); cur_cb_list_item = cur_cb_list_item->next; } - // FIXME: why does having this decref of args cause a segfault?! Py_DECREF(args); PyGILState_Release(state); return 0; @@ -228,10 +228,10 @@ static int PyXmlSec_ReadCB(void* context, char* buffer, int len) { int result; if (py_bytes_read && PyLong_Check(py_bytes_read)) { result = (int)PyLong_AsLong(py_bytes_read); - Py_DECREF(py_bytes_read); } else { result = EOF; } + Py_XDECREF(py_bytes_read); PyGILState_Release(state); return result; @@ -331,18 +331,22 @@ static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, } if (!PyCallable_Check(cb_list_item->match_cb)) { PyErr_SetString(PyExc_TypeError, "input_match_callback must be a callable"); + free(cb_list_item); return NULL; } if (!PyCallable_Check(cb_list_item->open_cb)) { PyErr_SetString(PyExc_TypeError, "input_open_callback must be a callable"); + free(cb_list_item); return NULL; } if (!PyCallable_Check(cb_list_item->read_cb)) { PyErr_SetString(PyExc_TypeError, "input_read_callback must be a callable"); + free(cb_list_item); return NULL; } if (!PyCallable_Check(cb_list_item->close_cb)) { PyErr_SetString(PyExc_TypeError, "input_close_callback must be a callable"); + free(cb_list_item); return NULL; } Py_INCREF(cb_list_item->match_cb); From 12690e865c91414bd2f60f5c4c1864408001396c Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Thu, 29 Jul 2021 10:37:17 +0100 Subject: [PATCH 048/211] Add signatures for new functions to __init__.pyi --- src/xmlsec/__init__.pyi | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 9ec147e9..93ff8658 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -1,5 +1,7 @@ import sys -from typing import AnyStr, IO, Iterable, Optional, Type, TypeVar, Union, overload +from typing import ( + Any, AnyStr, Callable, IO, Iterable, Optional, Type, TypeVar, Union, + overload) from lxml.etree import _Element @@ -24,6 +26,14 @@ _K = TypeVar('_K', bound=Key) def enable_debug_trace(enabled: bool = ...) -> None: ... def init() -> None: ... def shutdown() -> None: ... +def cleanup_callbacks() -> None: ... +def register_default_callbacks() -> None: ... +def register_callbacks( + input_match_callback: Callable[[bytes], bool], + input_open_callback: Callable[[bytes], Any], + input_read_callback: Callable[[Any, memoryview], int], + input_close_callback: Callable[[Any], None], +) -> None: ... @overload def base64_default_line_size() -> int: ... @overload From 91730b25986284304da480c62f4f1dbf6cd44de0 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Fri, 30 Jul 2021 11:02:00 +0100 Subject: [PATCH 049/211] Test and fix IO callback bindings Turns out the `xmlSecAllIOCallbacks` pointer list yields the stored callbacks in reverse, _and_ the default callbacks steal everything ( libxml2's `xmlFileMatch` is literally defined as `return(1)`! So we - Simplify our linked list of sets of Python callbacks to cons, rather than append - Don't bother trying to track interleaving default callbacks and Python callbacks, because `cur_cb_list_item` would get left in a bad state when the default matched. - Instead, drop all callbacks added prior to the default callbacks being added. Tests-wise, there are a lot of failures relating to unreferenced objects and/or memory leaks. However, as an example the top test is completely commented out right now (i.e. doesn't do anything with the library), so I'm a bit stuck as to how to grapple with that! --- src/main.c | 62 +++++++-------------- tests/test_main.py | 134 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 42 deletions(-) diff --git a/src/main.c b/src/main.c index 205174e6..41f5e5fc 100644 --- a/src/main.c +++ b/src/main.c @@ -146,47 +146,34 @@ typedef struct CbList { } CbList; static CbList* registered_callbacks = NULL; -static CbList* rcb_tail = NULL; -static void RCBListAppend(CbList* cb_list_item) { - if (registered_callbacks == NULL) { - registered_callbacks = cb_list_item; - } else { - rcb_tail->next = cb_list_item; - } - rcb_tail = cb_list_item; +static void RCBListCons(CbList* cb_list_item) { + cb_list_item->next = registered_callbacks; + registered_callbacks = cb_list_item; } static void RCBListClear() { CbList* cb_list_item = registered_callbacks; while (cb_list_item) { - Py_XDECREF(cb_list_item->match_cb); - Py_XDECREF(cb_list_item->open_cb); - Py_XDECREF(cb_list_item->read_cb); - Py_XDECREF(cb_list_item->close_cb); + Py_DECREF(cb_list_item->match_cb); + Py_DECREF(cb_list_item->open_cb); + Py_DECREF(cb_list_item->read_cb); + Py_DECREF(cb_list_item->close_cb); CbList* next = cb_list_item->next; free(cb_list_item); cb_list_item = next; } registered_callbacks = NULL; - rcb_tail = NULL; } // The currently executing set of Python callbacks: -static CbList* cur_cb_list_item = NULL; +static CbList* cur_cb_list_item; static int PyXmlSec_MatchCB(const char* filename) { - if (!cur_cb_list_item) { - cur_cb_list_item = registered_callbacks; - } - while (cur_cb_list_item && !cur_cb_list_item->match_cb) { - // Spool past any default callback placeholders executed since we were - // last called back: - cur_cb_list_item = cur_cb_list_item->next; - } + cur_cb_list_item = registered_callbacks; PyGILState_STATE state = PyGILState_Ensure(); PyObject* args = Py_BuildValue("(y)", filename); - while (cur_cb_list_item && cur_cb_list_item->match_cb) { + while (cur_cb_list_item) { PyObject* result = PyObject_CallObject(cur_cb_list_item->match_cb, args); if (result && PyObject_IsTrue(result)) { Py_DECREF(result); @@ -247,9 +234,6 @@ static int PyXmlSec_CloseCB(void* context) { Py_DECREF(result); PyGILState_Release(state); - // We reset `cur_cb_list_item` because we've finished processing the set of - // callbacks that was matched - cur_cb_list_item = NULL; return 0; } @@ -263,7 +247,7 @@ static PyObject* PyXmlSec_PyIOCleanupCallbacks(PyObject *self) { PyXmlSec_MatchCB, PyXmlSec_OpenCB, PyXmlSec_ReadCB, PyXmlSec_CloseCB) < 0) { return NULL; - }; + } RCBListClear(); Py_RETURN_NONE; } @@ -271,23 +255,17 @@ static PyObject* PyXmlSec_PyIOCleanupCallbacks(PyObject *self) { static char PyXmlSec_PyIORegisterDefaultCallbacks__doc__[] = \ "Register globally xmlsec's own default set of IO callbacks."; static PyObject* PyXmlSec_PyIORegisterDefaultCallbacks(PyObject *self) { + // NB: The default callbacks (specifically libxml2's `xmlFileMatch`) always + // match, and callbacks are called in the reverse order to that which they + // were added. So, there's no point in holding onto any previously registered + // callbacks, because they will never be run: + xmlSecIOCleanupCallbacks(); + RCBListClear(); if (xmlSecIORegisterDefaultCallbacks() < 0) { return NULL; } - // We place a nulled item on the callback list to represent whenever the - // default callbacks are going to be invoked: - CbList* cb_list_item = malloc(sizeof(CbList)); - if (cb_list_item == NULL) { - return NULL; - } - cb_list_item->match_cb = NULL; - cb_list_item->open_cb = NULL; - cb_list_item->read_cb = NULL; - cb_list_item->close_cb = NULL; - cb_list_item->next = NULL; - RCBListAppend(cb_list_item); - // We need to make sure we can continue trying to match futher Python - // callbacks if the default callback doesn't match: + // We need to make sure we can continue trying to match any newly added + // Python callbacks: if (xmlSecIORegisterCallbacks( PyXmlSec_MatchCB, PyXmlSec_OpenCB, PyXmlSec_ReadCB, PyXmlSec_CloseCB) < 0) { @@ -354,7 +332,7 @@ static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, Py_INCREF(cb_list_item->read_cb); Py_INCREF(cb_list_item->close_cb); cb_list_item->next = NULL; - RCBListAppend(cb_list_item); + RCBListCons(cb_list_item); // NB: We don't need to register the callbacks with `xmlsec` here, because // we've already registered our helper functions that will trawl through our // list of callbacks. diff --git a/tests/test_main.py b/tests/test_main.py index a0d144ca..ac19f3fc 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,10 @@ import xmlsec +from xmlsec import constants as consts + +from io import BytesIO + +from hypothesis import given, strategies +import pytest from tests import base @@ -30,3 +36,131 @@ def test_set_base64_default_line_size_rejects_negative_values(self): with self.assertRaises(ValueError): xmlsec.base64_default_line_size(-1) self.assertEqual(xmlsec.base64_default_line_size(), size) + + +class TestCallbacks(base.TestMemoryLeaks): + def setUp(self): + super().setUp() + xmlsec.cleanup_callbacks() + + @given(funcs=strategies.lists( + strategies.sampled_from([ + lambda: None + # xmlsec.cleanup_callbacks, + # xmlsec.register_default_callbacks, + ]) + )) + def test_arbitrary_cleaning_and_default_callback_registration(self, funcs): + # FIXME: This test seems to detelct unreferenced objects and memory + # leaks even if it never does anything! + pass + # for f in funcs: + # f() + + def _sign_doc(self): + root = self.load_xml("doc.xml") + sign = xmlsec.template.create( + root, + c14n_method=consts.TransformExclC14N, + sign_method=consts.TransformRsaSha1 + ) + xmlsec.template.add_reference( + sign, consts.TransformSha1, uri="cid:123456") + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file( + self.path("rsakey.pem"), format=consts.KeyDataFormatPem + ) + ctx.sign(sign) + return sign + + def _expect_sign_failure(self): + exc_info = pytest.raises(xmlsec.Error, self._sign_doc) + self.assertEqual(exc_info.value.args, (1, 'failed to sign')) + + def _register_mismatch_callbacks(self, match_cb=lambda filename: False): + xmlsec.register_callbacks( + match_cb, + lambda filename: None, + lambda none, buf: 0, + lambda none: None, + ) + + def _register_match_callbacks(self): + xmlsec.register_callbacks( + lambda filename: filename == b'cid:123456', + lambda filename: BytesIO(b''), + lambda bio, buf: bio.readinto(buf), + lambda bio: bio.close(), + ) + + def _find(self, elem, *tags): + for tag in tags: + elem = elem.find( + '{{http://www.w3.org/2000/09/xmldsig#}}{}'.format(tag)) + return elem + + def _verify_external_data_signature(self): + signature = self._sign_doc() + digest = self._find( + signature, 'SignedInfo', 'Reference', 'DigestValue' + ).text + self.assertEqual(digest, 'VihZwVMGJ48NsNl7ertVHiURXk8=') + + def test_sign_external_data_no_callbacks_fails(self): + self._expect_sign_failure() + + def test_sign_external_data_default_callbacks_fails(self): + xmlsec.register_default_callbacks() + self._expect_sign_failure() + + def test_sign_external_data_no_matching_callbacks_fails(self): + self._register_mismatch_callbacks() + self._expect_sign_failure() + + def test_sign_data_from_callbacks(self): + self._register_match_callbacks() + self._verify_external_data_signature() + + @given( + num_prior_mismatches=strategies.integers(min_value=1, max_value=50), + num_post_mismatches=strategies.integers(min_value=1, max_value=50), + ) + def test_sign_data_not_first_callback( + self, num_prior_mismatches, num_post_mismatches + ): + bad_match_calls = 0 + + def match_cb(filename): + nonlocal bad_match_calls + bad_match_calls += 1 + False + + for _ in range(num_post_mismatches): + self._register_mismatch_callbacks(match_cb) + + self._register_match_callbacks() + + for _ in range(num_prior_mismatches): + self._register_mismatch_callbacks() + + self._verify_external_data_signature() + self.assertEqual(bad_match_calls, 0) + + def test_failed_sign_because_default_callbacks(self): + mismatch_calls = 0 + + def mismatch_cb(filename): + nonlocal mismatch_calls + mismatch_calls += 1 + False + + # NB: These first two sets of callbacks should never get called, + # because the default callbacks always match beforehand: + self._register_match_callbacks() + self._register_mismatch_callbacks(mismatch_cb) + xmlsec.register_default_callbacks() + self._register_mismatch_callbacks(mismatch_cb) + self._register_mismatch_callbacks(mismatch_cb) + self._expect_sign_failure() + self.assertEqual(mismatch_calls, 2) From 5f65a37285e07432b70f13337eaa47706a9b5728 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Mon, 2 Aug 2021 15:26:42 +0100 Subject: [PATCH 050/211] Remove hypothesis from callback testing --- tests/test_main.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index ac19f3fc..c57b6a9f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,6 @@ from io import BytesIO -from hypothesis import given, strategies import pytest from tests import base @@ -43,20 +42,6 @@ def setUp(self): super().setUp() xmlsec.cleanup_callbacks() - @given(funcs=strategies.lists( - strategies.sampled_from([ - lambda: None - # xmlsec.cleanup_callbacks, - # xmlsec.register_default_callbacks, - ]) - )) - def test_arbitrary_cleaning_and_default_callback_registration(self, funcs): - # FIXME: This test seems to detelct unreferenced objects and memory - # leaks even if it never does anything! - pass - # for f in funcs: - # f() - def _sign_doc(self): root = self.load_xml("doc.xml") sign = xmlsec.template.create( @@ -122,13 +107,7 @@ def test_sign_data_from_callbacks(self): self._register_match_callbacks() self._verify_external_data_signature() - @given( - num_prior_mismatches=strategies.integers(min_value=1, max_value=50), - num_post_mismatches=strategies.integers(min_value=1, max_value=50), - ) - def test_sign_data_not_first_callback( - self, num_prior_mismatches, num_post_mismatches - ): + def test_sign_data_not_first_callback(self): bad_match_calls = 0 def match_cb(filename): @@ -136,12 +115,12 @@ def match_cb(filename): bad_match_calls += 1 False - for _ in range(num_post_mismatches): + for _ in range(2): self._register_mismatch_callbacks(match_cb) self._register_match_callbacks() - for _ in range(num_prior_mismatches): + for _ in range(2): self._register_mismatch_callbacks() self._verify_external_data_signature() From a2c2b7582cad64a201688a2d67b2f62bd8846257 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Mon, 16 Aug 2021 16:38:51 +0100 Subject: [PATCH 051/211] Use unittest's exception assertion because pytest.raises leaves junk around! --- tests/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index c57b6a9f..40889621 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -60,8 +60,8 @@ def _sign_doc(self): return sign def _expect_sign_failure(self): - exc_info = pytest.raises(xmlsec.Error, self._sign_doc) - self.assertEqual(exc_info.value.args, (1, 'failed to sign')) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + self._sign_doc() def _register_mismatch_callbacks(self, match_cb=lambda filename: False): xmlsec.register_callbacks( From ce90c06a32556cf79b0fc224b9ebe63d3d08c8e8 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Fri, 20 Aug 2021 10:48:38 +0100 Subject: [PATCH 052/211] Remove unused import --- tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 40889621..556e0161 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,9 @@ +from lxml import etree import xmlsec from xmlsec import constants as consts from io import BytesIO -import pytest from tests import base From f17f4b9c957a18e9210c72162f06cfa7414b086e Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Fri, 20 Aug 2021 10:49:31 +0100 Subject: [PATCH 053/211] Use xpath to traverse signature structure in tests It seems to be way more robust across library versions. --- tests/test_main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 556e0161..2c57f3e4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -80,10 +80,15 @@ def _register_match_callbacks(self): ) def _find(self, elem, *tags): - for tag in tags: - elem = elem.find( - '{{http://www.w3.org/2000/09/xmldsig#}}{}'.format(tag)) - return elem + try: + return elem.xpath( + './' + '/'.join('xmldsig:{}'.format(tag) for tag in tags), + namespaces={ + 'xmldsig': 'http://www.w3.org/2000/09/xmldsig#', + } + )[0] + except IndexError as e: + raise KeyError(tags) from e def _verify_external_data_signature(self): signature = self._sign_doc() From 7b41f2710e923e6f51087df9d05039d2f0b66369 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Mon, 23 Aug 2021 10:13:33 +0100 Subject: [PATCH 054/211] Remove unused import --- tests/test_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 2c57f3e4..c3271139 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,3 @@ -from lxml import etree import xmlsec from xmlsec import constants as consts From 9798d6d99ebf502152f5432589a50bc82fd69264 Mon Sep 17 00:00:00 2001 From: Paul Weaver Date: Mon, 23 Aug 2021 10:13:50 +0100 Subject: [PATCH 055/211] Test TypeError when setting non-callable callbacks --- tests/test_main.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index c3271139..648aa5b0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -62,13 +62,16 @@ def _expect_sign_failure(self): with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): self._sign_doc() - def _register_mismatch_callbacks(self, match_cb=lambda filename: False): - xmlsec.register_callbacks( + def _mismatch_callbacks(self, match_cb=lambda filename: False): + return [ match_cb, lambda filename: None, lambda none, buf: 0, lambda none: None, - ) + ] + + def _register_mismatch_callbacks(self, match_cb=lambda filename: False): + xmlsec.register_callbacks(*self._mismatch_callbacks(match_cb)) def _register_match_callbacks(self): xmlsec.register_callbacks( @@ -147,3 +150,9 @@ def mismatch_cb(filename): self._register_mismatch_callbacks(mismatch_cb) self._expect_sign_failure() self.assertEqual(mismatch_calls, 2) + + def test_register_non_callables(self): + for idx in range(4): + cbs = self._mismatch_callbacks() + cbs[idx] = None + self.assertRaises(TypeError, xmlsec.register_callbacks, *cbs) From fc565ba2b82439a12e7bed23103f222bde860b6e Mon Sep 17 00:00:00 2001 From: Oleg Hoefling Date: Sun, 5 Sep 2021 18:20:13 +0200 Subject: [PATCH 056/211] coverage increase: test for graceful failing on read callback wrong return type Signed-off-by: Oleg Hoefling --- tests/test_main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 648aa5b0..69e8a26d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -156,3 +156,12 @@ def test_register_non_callables(self): cbs = self._mismatch_callbacks() cbs[idx] = None self.assertRaises(TypeError, xmlsec.register_callbacks, *cbs) + + def test_sign_external_data_fails_on_read_callback_wrong_returns(self): + xmlsec.register_callbacks( + lambda filename: filename == b'cid:123456', + lambda filename: BytesIO(b''), + lambda bio, buf: None, + lambda bio: bio.close(), + ) + self._expect_sign_failure() From f18946b1c9ffd1d030b435dd8f40f85d842e5a5c Mon Sep 17 00:00:00 2001 From: Oleg Hoefling Date: Sun, 5 Sep 2021 18:31:59 +0200 Subject: [PATCH 057/211] skip test failing on windows for now Signed-off-by: Oleg Hoefling --- tests/test_main.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 69e8a26d..9fb71eaf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,10 @@ -import xmlsec -from xmlsec import constants as consts - +import sys from io import BytesIO +from unittest import skipIf +import xmlsec from tests import base +from xmlsec import constants as consts class TestBase64LineSize(base.TestMemoryLeaks): @@ -43,18 +44,11 @@ def setUp(self): def _sign_doc(self): root = self.load_xml("doc.xml") - sign = xmlsec.template.create( - root, - c14n_method=consts.TransformExclC14N, - sign_method=consts.TransformRsaSha1 - ) - xmlsec.template.add_reference( - sign, consts.TransformSha1, uri="cid:123456") + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + xmlsec.template.add_reference(sign, consts.TransformSha1, uri="cid:123456") ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file( - self.path("rsakey.pem"), format=consts.KeyDataFormatPem - ) + ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) ctx.sign(sign) return sign @@ -87,16 +81,14 @@ def _find(self, elem, *tags): './' + '/'.join('xmldsig:{}'.format(tag) for tag in tags), namespaces={ 'xmldsig': 'http://www.w3.org/2000/09/xmldsig#', - } + }, )[0] except IndexError as e: raise KeyError(tags) from e def _verify_external_data_signature(self): signature = self._sign_doc() - digest = self._find( - signature, 'SignedInfo', 'Reference', 'DigestValue' - ).text + digest = self._find(signature, 'SignedInfo', 'Reference', 'DigestValue').text self.assertEqual(digest, 'VihZwVMGJ48NsNl7ertVHiURXk8=') def test_sign_external_data_no_callbacks_fails(self): @@ -133,6 +125,7 @@ def match_cb(filename): self._verify_external_data_signature() self.assertEqual(bad_match_calls, 0) + @skipIf(sys.platform == "win32", "unclear behaviour on windows") def test_failed_sign_because_default_callbacks(self): mismatch_calls = 0 From cf38b52c81d80897500f153ae5b9307439d6145a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 13 Dec 2021 10:31:24 +0000 Subject: [PATCH 058/211] exclude lxml 4.7 because the wheels are missing the lxml includes see https://bugs.launchpad.net/lxml/+bug/1954644 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6c69db80..9262cbf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,4 +23,4 @@ known_first_party = ['xmlsec'] known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] [build-system] -requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4'] +requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] diff --git a/requirements.txt b/requirements.txt index 827d75e0..20a5dfb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ pkgconfig -lxml >= 3.8.0 +lxml >= 3.8.0, !=4.7.0 From 9acc1538e3ed7522ab9237ea40ef4abc15d15445 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 13 Dec 2021 11:19:26 +0000 Subject: [PATCH 059/211] remove pkgconfig as it's only a build-system requirement --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 20a5dfb7..014bfe10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -pkgconfig lxml >= 3.8.0, !=4.7.0 From 154c51df4769d35a51cb2ac48c5f606772d42df8 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 13 Dec 2021 11:22:58 +0000 Subject: [PATCH 060/211] use build to build wheels --- .github/workflows/linuxbrew.yml | 4 ++-- .github/workflows/macosx.yml | 4 ++-- .github/workflows/manylinux2010.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 6502b830..0d85caee 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -18,11 +18,11 @@ jobs: echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile brew update brew install python gcc libxml2 libxmlsec1 pkg-config - pip3 install --upgrade setuptools wheel + pip3 install --upgrade setuptools wheel build ln -s $(brew --prefix)/bin/gcc-11 $(brew --prefix)/bin/gcc-5 - name: Build linux_x86_64 wheel run: | - python3 setup.py bdist_wheel + python3 -m build rm -rf build/ - name: Install test dependencies run: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 70e00291..b5921dc5 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -14,7 +14,7 @@ jobs: python-version: ${{ matrix.python }} - name: Install build dependencies run: | - pip install --upgrade pip setuptools wheel + pip install --upgrade pip setuptools wheel build brew install libxml2 libxmlsec1 pkg-config - name: Build macosx_x86_64 wheel env: @@ -22,7 +22,7 @@ jobs: CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" run: | - python setup.py bdist_wheel + python -m build rm -rf build/ - name: Set environment variables shell: bash diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index e2b1f449..c357db52 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v1 - name: Install build dependencies run: | - /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel + /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build - name: Set environment variables shell: bash run: | @@ -22,7 +22,7 @@ jobs: # disable libxml2-2.9.12 because of https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 PYXMLSEC_LIBXML2_VERSION: 2.9.10 run: | - /opt/python/${{ matrix.python-abi }}/bin/python setup.py bdist_wheel + /opt/python/${{ matrix.python-abi }}/bin/python -m build - name: Label manylinux2010_x86_64 wheel run: | ls -la dist/ From a65be810213327343c197384a2cb648654ab3d6d Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Thu, 10 Feb 2022 19:22:38 +0300 Subject: [PATCH 061/211] Switch on modern headers lxml --- src/lxml.c | 2 +- src/lxml.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 862ee682..aa1abae0 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -11,7 +11,7 @@ #include "lxml.h" #include -#include +#include #include #include diff --git a/src/lxml.h b/src/lxml.h index 435ccdff..6824076b 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -16,7 +16,7 @@ #include #include -#include +#include typedef struct LxmlElement* PyXmlSec_LxmlElementPtr; typedef struct LxmlDocument* PyXmlSec_LxmlDocumentPtr; From bf6984a555f02580dd40f865997c0d9b61d81fa6 Mon Sep 17 00:00:00 2001 From: AbdealiJK Date: Tue, 22 Mar 2022 16:39:39 +0530 Subject: [PATCH 062/211] setup.py: Fix typo in PYXMLSEC_LIBXSLT_VERSION --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 541eb029..45d8e664 100644 --- a/setup.py +++ b/setup.py @@ -168,7 +168,7 @@ def prepare_static_build_linux(self): self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1g') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION', '1.16') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', None) - self.libxslt_version = os.environ.get('PYXMLSEC_LIBXLST_VERSION', None) + self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION', None) self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.2.11') self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.2.31') From 5c6bbdb86338ed7ece3cfec017295953a1527b36 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 12:38:37 +0200 Subject: [PATCH 063/211] renew sources for source dists Signed-off-by: oleg.hoefling --- setup.py | 155 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index 45d8e664..9dbf069d 100644 --- a/setup.py +++ b/setup.py @@ -1,20 +1,94 @@ +import contextlib +import html.parser import io +import json import multiprocessing import os +import re import subprocess import sys import tarfile import zipfile from distutils import log +from distutils.version import StrictVersion as Version from distutils.errors import DistutilsError from pathlib import Path -from urllib.request import urlcleanup, urljoin, urlretrieve +from urllib.request import urlcleanup, urljoin, urlopen, urlretrieve from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig -class build_ext(build_ext_orig, object): +class HrefCollector(html.parser.HTMLParser): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for name, value in attrs: + if name == 'href': + self.hrefs.append(value) + + +def latest_release_from_html(url, matcher): + with contextlib.closing(urlopen(url)) as r: + charset = r.headers.get_content_charset() or 'utf-8' + content = r.read().decode(charset) + collector = HrefCollector() + collector.feed(content) + hrefs = collector.hrefs + + def comp(text): + try: + return Version(matcher.match(text).groupdict()['version']) + except (AttributeError, ValueError): + return Version('0.0') + + latest = max(hrefs, key=comp) + return '{}/{}'.format(url, latest) + + +def latest_release_from_gnome_org_cache(url, lib_name): + cache_url = '{}/cache.json'.format(url) + with contextlib.closing(urlopen(cache_url)) as r: + cache = json.load(r) + latest_version = cache[2][lib_name][-1] + latest_source = cache[1][lib_name][latest_version]['tar.xz'] + return '{}/{}'.format(url, latest_source) + + +def latest_zlib_release(): + return latest_release_from_html( + 'https://zlib.net/fossils', re.compile('zlib-(?P.*).tar.gz') + ) + + +def latest_libiconv_release(): + return latest_release_from_html( + 'https://ftp.gnu.org/pub/gnu/libiconv', re.compile('libiconv-(?P.*).tar.gz') + ) + + +def latest_libxml2_release(): + return latest_release_from_gnome_org_cache( + 'https://download.gnome.org/sources/libxml2', 'libxml2' + ) + + +def latest_libxslt_release(): + return latest_release_from_gnome_org_cache( + 'https://download.gnome.org/sources/libxslt', 'libxslt' + ) + + +def latest_xmlsec_release(): + return latest_release_from_html( + 'https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz') + ) + + +class build_ext(build_ext_orig): def info(self, message): self.announce(message, level=log.INFO) @@ -36,6 +110,7 @@ def run(self): self.libs_dir = Path(os.environ.get('PYXMLSEC_LIBS_DIR', 'libs')) self.libs_dir.mkdir(exist_ok=True) + self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) if sys.platform == 'win32': self.prepare_static_build_win() @@ -165,71 +240,85 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1g') - self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION', '1.16') - self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', None) - self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION', None) - self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.2.11') - self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.2.31') - - self.info('Settings:') - self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) - self.info('{:20} {}'.format('zlib version:', self.zlib_version)) - self.info('{:20} {}'.format('libiconv version:', self.libiconv_version)) - self.info('{:20} {}'.format('libxml2 version:', self.libxml2_version or 'unset, using latest')) - self.info('{:20} {}'.format('libxslt version:', self.libxslt_version or 'unset, using latest')) - self.info('{:20} {}'.format('xmlsec1 version:', self.xmlsec1_version)) + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1q') + self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') + self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') + self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') + self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION') + self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION') # fetch openssl openssl_tar = next(self.libs_dir.glob('openssl*.tar.gz'), None) if openssl_tar is None: - self.info('OpenSSL source tar not found, downloading ...') + self.info('{:10}: {}'.format('OpenSSL', 'source tar not found, downloading ...')) openssl_tar = self.libs_dir / 'openssl.tar.gz' + self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) urlretrieve('https://www.openssl.org/source/openssl-{}.tar.gz'.format(self.openssl_version), str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) if zlib_tar is None: - self.info('zlib source tar not found, downloading ...') + self.info('{:10}: {}'.format('zlib', 'source not found, downloading ...')) zlib_tar = self.libs_dir / 'zlib.tar.gz' - urlretrieve('https://zlib.net/zlib-{}.tar.gz'.format(self.zlib_version), str(zlib_tar)) + if self.zlib_version is None: + url = latest_zlib_release() + self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://zlib.net/fossils/zlib-{}.tar.gz'.format(self.zlib_version) + self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION={}, downloading from {}'.format(self.zlib_version, url))) + urlretrieve(url, str(zlib_tar)) # fetch libiconv libiconv_tar = next(self.libs_dir.glob('libiconv*.tar.gz'), None) if libiconv_tar is None: - self.info('libiconv source tar not found, downloading ...') + self.info('{:10}: {}'.format('libiconv', 'source not found, downloading ...')) libiconv_tar = self.libs_dir / 'libiconv.tar.gz' - urlretrieve( - 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{}.tar.gz'.format(self.libiconv_version), str(libiconv_tar) - ) + if self.libiconv_version is None: + url = latest_libiconv_release() + self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{}.tar.gz'.format(self.libiconv_version) + self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_LIBICONV_VERSION={}, downloading from {}'.format(self.libiconv_version, url))) + urlretrieve(url, str(libiconv_tar)) # fetch libxml2 - libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.gz'), None) + libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.xz'), None) if libxml2_tar is None: - self.info('Libxml2 source tar not found, downloading ...') + self.info('{:10}: {}'.format('libxml2', 'source tar not found, downloading ...')) if self.libxml2_version is None: - url = 'http://xmlsoft.org/sources/LATEST_LIBXML2' + url = latest_libxml2_release() + self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {}'.format(url))) else: - url = 'http://xmlsoft.org/sources/libxml2-{}.tar.gz'.format(self.libxml2_version) - libxml2_tar = self.libs_dir / 'libxml2.tar.gz' + version_prefix, _ = self.libxml2_version.split('.', -1) + url = 'https://download.gnome.org/sources/libxml2/{}/libxml2-{}.tar.xz'.format(version_prefix, self.libxml2_version) + self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION={}, downloading from {}'.format(self.libxml2_version, url))) + libxml2_tar = self.libs_dir / 'libxml2.tar.xz' urlretrieve(url, str(libxml2_tar)) # fetch libxslt libxslt_tar = next(self.libs_dir.glob('libxslt*.tar.gz'), None) if libxslt_tar is None: - self.info('libxslt source tar not found, downloading ...') + self.info('{:10}: {}'.format('libxslt', 'source tar not found, downloading ...')) if self.libxslt_version is None: - url = 'http://xmlsoft.org/sources/LATEST_LIBXSLT' + url = latest_libxslt_release() + self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {}'.format(url))) else: - url = 'http://xmlsoft.org/sources/libxslt-{}.tar.gz'.format(self.libxslt_version) + version_prefix, _ = self.libxslt_version.split('.', -1) + url = 'https://download.gnome.org/sources/libxslt/{}/libxslt-{}.tar.xz'.format(version_prefix, self.libxslt_version) + self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION={}, downloading from {}'.format(self.libxslt_version, url))) libxslt_tar = self.libs_dir / 'libxslt.tar.gz' urlretrieve(url, str(libxslt_tar)) # fetch xmlsec1 xmlsec1_tar = next(self.libs_dir.glob('xmlsec1*.tar.gz'), None) if xmlsec1_tar is None: - self.info('xmlsec1 source tar not found, downloading ...') - url = 'http://www.aleksey.com/xmlsec/download/xmlsec1-{}.tar.gz'.format(self.xmlsec1_version) + self.info('{:10}: {}'.format('xmlsec1', 'source tar not found, downloading ...')) + if self.xmlsec1_version is None: + url = latest_xmlsec_release() + self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://www.aleksey.com/xmlsec/download/xmlsec1-{}.tar.gz'.format(self.xmlsec1_version) + self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION={}, downloading from {}'.format(self.xmlsec1_version, url))) xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' urlretrieve(url, str(xmlsec1_tar)) From 811c5cca2522283e96773cc78de3736049b5b020 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 12:40:30 +0200 Subject: [PATCH 064/211] remove libxml2 pin in manylinux2010, sdist jobs Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 2 -- .github/workflows/sdist.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index c357db52..99a81a96 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -19,8 +19,6 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true - # disable libxml2-2.9.12 because of https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 - PYXMLSEC_LIBXML2_VERSION: 2.9.10 run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build - name: Label manylinux2010_x86_64 wheel diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index d6cf6c9d..5b5dac96 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -18,8 +18,6 @@ jobs: - name: Install test dependencies env: PYXMLSEC_STATIC_DEPS: true - # disable libxml2-2.9.12 because of https://gitlab.gnome.org/GNOME/libxslt/-/issues/52 - PYXMLSEC_LIBXML2_VERSION: 2.9.10 run: | pip install --upgrade -r requirements-test.txt pip install dist/xmlsec-$(python setup.py --version).tar.gz From 1858a467b5d7ea4ca95d15717b339e492380c134 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 13:05:28 +0200 Subject: [PATCH 065/211] bump github actions versions Signed-off-by: oleg.hoefling --- .github/workflows/linuxbrew.yml | 4 ++-- .github/workflows/macosx.yml | 4 ++-- .github/workflows/manylinux2010.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 0d85caee..4d2f5ba4 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -4,9 +4,9 @@ jobs: linuxbrew: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install build dependencies diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index b5921dc5..ef72fb78 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -7,9 +7,9 @@ jobs: matrix: python: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install build dependencies diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 99a81a96..49059d84 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -8,7 +8,7 @@ jobs: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Install build dependencies run: | /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 5b5dac96..6082fcd6 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -4,9 +4,9 @@ jobs: sdist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install build dependencies From 8393c7a967b68ed34f4921d33e4d5f77cb40a803 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 13:10:24 +0200 Subject: [PATCH 066/211] workaround for https://github.com/actions/runner/issues/2033 Signed-off-by: oleg.hoefling --- .github/workflows/manylinux2010.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux2010.yml index 49059d84..d1769b74 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux2010.yml @@ -8,9 +8,11 @@ jobs: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v1 - name: Install build dependencies run: | + # https://github.com/actions/runner/issues/2033 + chown -R $(id -u):$(id -g) $PWD /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build - name: Set environment variables shell: bash From 8551a4f425bed5c24c26fd0c4e03a42ecd2d2b4c Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 13:27:42 +0200 Subject: [PATCH 067/211] bump gcc-5 symlink in linuxbrew job Signed-off-by: oleg.hoefling --- .github/workflows/linuxbrew.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 4d2f5ba4..191e2001 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -19,7 +19,8 @@ jobs: brew update brew install python gcc libxml2 libxmlsec1 pkg-config pip3 install --upgrade setuptools wheel build - ln -s $(brew --prefix)/bin/gcc-11 $(brew --prefix)/bin/gcc-5 + ln -s $(brew --prefix)/bin/gcc-12 $(brew --prefix)/bin/gcc-5 + ls -l $(brew --prefix)/bin/gcc* - name: Build linux_x86_64 wheel run: | python3 -m build From 3ebbed8f0cc46c17f13c2e9a6ad98888c3e406c6 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 13:32:26 +0200 Subject: [PATCH 068/211] bump python 3.9 to latest patch in appveyor ci Signed-off-by: oleg.hoefling --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 54bd2b1e..9155acc0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,9 +9,9 @@ environment: - python: 38 - python: 38-x64 - python: 39 - python_version: 3.9.0 + python_version: 3.9.13 - python: 39-x64 - python_version: 3.9.0 + python_version: 3.9.13 install: - ps: | From bc30efd49dab62a9c96f44f0dc31a4de4181dabd Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 6 May 2022 14:52:01 +0200 Subject: [PATCH 069/211] Resolve key loading issue on big-endian systems. Closes: #208 --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index 46dbf651..1362b128 100644 --- a/src/keys.c +++ b/src/keys.c @@ -142,7 +142,7 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* Py_ssize_t data_size = 0; PYXMLSEC_DEBUG("load key from file - start"); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OH|z:from_file", kwlist, &file, &format, &password)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OI|z:from_file", kwlist, &file, &format, &password)) { goto ON_FAIL; } From b04a76285188e886e61bbb6540466fc3b5738e39 Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Mon, 28 Mar 2022 11:59:00 +0200 Subject: [PATCH 070/211] xmlsec workaround for gh#mehcode/python-xmlsec#84 Fixes #96 --- tests/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/base.py b/tests/base.py index e834f080..b6b3b56a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -99,10 +99,11 @@ def load(self, name): def load_xml(self, name, xpath=None): """returns xml.etree""" - root = etree.parse(self.path(name)).getroot() - if xpath is None: - return root - return root.find(xpath) + with open(self.path(name)) as f: + root = etree.parse(f).getroot() + if xpath is None: + return root + return root.find(xpath) def dump(self, root): print(etree.tostring(root)) From ceaea9c9ad27bcb82d9b1a30cea83d96d34e2639 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 14:52:19 +0200 Subject: [PATCH 071/211] regenerate stubs, amend according to flake8-pyi, ensure stub generation test is running in the CI Signed-off-by: oleg.hoefling --- .appveyor.yml | 1 + .github/workflows/sdist.yml | 1 + pyproject.toml | 2 +- setup.cfg | 3 + src/xmlsec/__init__.pyi | 57 +++----- src/xmlsec/constants.pyi | 270 +++++++++++++++++------------------- src/xmlsec/template.pyi | 29 ++-- src/xmlsec/tree.pyi | 15 +- tests/test_type_stubs.py | 19 +-- 9 files changed, 188 insertions(+), 209 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9155acc0..f7b9e16d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -44,6 +44,7 @@ build_script: test: off test_script: - pip install -r requirements-test.txt + - pip install black # for stub generation tests - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist - pytest -v --color=yes --junitxml=unittests.xml - ps: Get-ChildItem dist\*.whl | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 6082fcd6..28fe79fe 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -20,6 +20,7 @@ jobs: PYXMLSEC_STATIC_DEPS: true run: | pip install --upgrade -r requirements-test.txt + pip install black # for stub generation tests pip install dist/xmlsec-$(python setup.py --version).tar.gz - name: Run tests run: | diff --git a/pyproject.toml b/pyproject.toml index 9262cbf5..2bfc3d21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.black] line_length = 130 skip-string-normalization = true -target_version = ['py38'] +target_version = ['py39'] include = '\.pyi?$' exclude = ''' diff --git a/setup.cfg b/setup.cfg index c61fd65b..88d815b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,4 +16,7 @@ all_files = 1 upload_dir = doc/build/html [flake8] +per-file-ignores = + *.pyi: E301, E302, E305, E501, E701, F401, F822 +exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* max-line-length = 130 diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 93ff8658..43959222 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -1,27 +1,12 @@ -import sys -from typing import ( - Any, AnyStr, Callable, IO, Iterable, Optional, Type, TypeVar, Union, - overload) +from collections.abc import Callable, Iterable +from typing import Any, AnyStr, IO, typeVar, overload +from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element -from xmlsec import constants, template, tree from xmlsec.constants import __KeyData as KeyData, __Transform as Transform -if sys.version_info >= (3, 6): - from os import PathLike - from pathlib import PurePath - - _Path = Union[str, bytes, PurePath, PathLike[str], PathLike[bytes]] -elif sys.version_info >= (3, 4): - from pathlib import PurePath - - _Path = Union[str, bytes, PurePath] -else: - _Path = Union[str, bytes] - -_E = TypeVar('_E', bound=_Element) -_K = TypeVar('_K', bound=Key) +_E = typeVar('_E', bound=_Element) def enable_debug_trace(enabled: bool = ...) -> None: ... def init() -> None: ... @@ -29,10 +14,10 @@ def shutdown() -> None: ... def cleanup_callbacks() -> None: ... def register_default_callbacks() -> None: ... def register_callbacks( - input_match_callback: Callable[[bytes], bool], - input_open_callback: Callable[[bytes], Any], - input_read_callback: Callable[[Any, memoryview], int], - input_close_callback: Callable[[Any], None], + input_match_callback: Callable[[bytes], bool], + input_open_callback: Callable[[bytes], Any], + input_read_callback: Callable[[Any, memoryview], int], + input_close_callback: Callable[[Any], None], ) -> None: ... @overload def base64_default_line_size() -> int: ... @@ -40,8 +25,8 @@ def base64_default_line_size() -> int: ... def base64_default_line_size(size: int) -> None: ... class EncryptionContext: - key: Optional[Key] - def __init__(self, manager: Optional[KeysManager] = None) -> None: ... + key: Key | None + def __init__(self, manager: KeysManager | None = ...) -> None: ... def decrypt(self, node: _Element) -> _Element: ... def encrypt_binary(self, template: _E, data: bytes) -> _E: ... def encrypt_uri(self, template: _E, uri: str) -> _E: ... @@ -54,30 +39,30 @@ class InternalError(Error): ... class Key: name: str @classmethod - def from_binary_data(cls: Type[_K], klass: KeyData, data: AnyStr) -> _K: ... + def from_binary_data(cls: type[Self], klass: KeyData, data: AnyStr) -> Self: ... @classmethod - def from_binary_file(cls: Type[_K], klass: KeyData, filename: _Path) -> _K: ... + def from_binary_file(cls: type[Self], klass: KeyData, filename: StrOrBytesPath) -> Self: ... @classmethod - def from_file(cls: Type[_K], file: Union[_Path, IO[AnyStr]], format: int, password: Optional[str] = ...) -> _K: ... + def from_file(cls: type[Self], file: GenericPath | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod - def from_memory(cls: Type[_K], data: AnyStr, format: int, password: Optional[str] = ...) -> _K: ... + def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod - def generate(cls: Type[_K], klass: KeyData, size: int, type: int) -> _K: ... - def load_cert_from_file(self, file: Union[_Path, IO[AnyStr]], format: int) -> None: ... + def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... + def load_cert_from_file(self, file: GenericPath | IO[AnyStr], format: int) -> None: ... def load_cert_from_memory(self, data: AnyStr, format: int) -> None: ... - def __copy__(self: _K) -> _K: ... - def __deepcopy__(self: _K) -> _K: ... + def __copy__(self: Self) -> Self: ... + def __deepcopy__(self: Self) -> Self: ... class KeysManager: def add_key(self, key: Key) -> None: ... - def load_cert(self, filename: _Path, format: int, type: int) -> None: ... + def load_cert(self, filename: StrOrBytesPath, format: int, type: int) -> None: ... def load_cert_from_memory(self, data: AnyStr, format: int, type: int) -> None: ... class SignatureContext: - key: Optional[Key] + key: Key | None def enable_reference_transform(self, transform: Transform) -> None: ... def enable_signature_transform(self, transform: Transform) -> None: ... - def register_id(self, node: _Element, id_attr: str = "ID", id_ns: Optional[str] = None) -> None: ... + def register_id(self, node: _Element, id_attr: str = ..., id_ns: str | None = ...) -> None: ... def set_enabled_key_data(self, keydata_list: Iterable[KeyData]) -> None: ... def sign(self, node: _Element) -> None: ... def sign_binary(self, bytes: bytes, transform: Transform) -> bytes: ... diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 97e00544..8e2165d8 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -1,5 +1,5 @@ import sys -from typing import NamedTuple, Optional +from typing import NamedTuple if sys.version_info >= (3, 8): from typing import Final @@ -7,147 +7,139 @@ else: from typing_extensions import Final class __KeyData(NamedTuple): # __KeyData type + href: str | None name: str - href: Optional[str] class __Transform(NamedTuple): # __Transform type + href: str | None name: str - href: Optional[str] usage: int -DSigNs: Final = 'http://www.w3.org/2000/09/xmldsig#' -EncNs: Final = 'http://www.w3.org/2001/04/xmlenc#' -KeyDataAes: Final = __KeyData('aes', 'http://www.aleksey.com/xmlsec/2002#AESKeyValue') -KeyDataDes: Final = __KeyData('des', 'http://www.aleksey.com/xmlsec/2002#DESKeyValue') -KeyDataDsa: Final = __KeyData('dsa', 'http://www.w3.org/2000/09/xmldsig#DSAKeyValue') -KeyDataEcdsa: Final = __KeyData('ecdsa', 'http://scap.nist.gov/specifications/tmsad/#resource-1.0') -KeyDataEncryptedKey: Final = __KeyData('enc-key', 'http://www.w3.org/2001/04/xmlenc#EncryptedKey') -KeyDataFormatBinary: Final = 1 -KeyDataFormatCertDer: Final = 8 -KeyDataFormatCertPem: Final = 7 -KeyDataFormatDer: Final = 3 -KeyDataFormatPem: Final = 2 -KeyDataFormatPkcs12: Final = 6 -KeyDataFormatPkcs8Der: Final = 5 -KeyDataFormatPkcs8Pem: Final = 4 -KeyDataFormatUnknown: Final = 0 -KeyDataHmac: Final = __KeyData('hmac', 'http://www.aleksey.com/xmlsec/2002#HMACKeyValue') -KeyDataName: Final = __KeyData('key-name', None) -KeyDataRawX509Cert: Final = __KeyData('raw-x509-cert', 'http://www.w3.org/2000/09/xmldsig#rawX509Certificate') -KeyDataRetrievalMethod: Final = __KeyData('retrieval-method', None) -KeyDataRsa: Final = __KeyData('rsa', 'http://www.w3.org/2000/09/xmldsig#RSAKeyValue') -KeyDataTypeAny: Final = 65535 -KeyDataTypeNone: Final = 0 -KeyDataTypePermanent: Final = 16 -KeyDataTypePrivate: Final = 2 -KeyDataTypePublic: Final = 1 -KeyDataTypeSession: Final = 8 -KeyDataTypeSymmetric: Final = 4 -KeyDataTypeTrusted: Final = 256 -KeyDataTypeUnknown: Final = 0 -KeyDataValue: Final = __KeyData('key-value', None) -KeyDataX509: Final = __KeyData('x509', 'http://www.w3.org/2000/09/xmldsig#X509Data') -NodeCanonicalizationMethod: Final = 'CanonicalizationMethod' -NodeCipherData: Final = 'CipherData' -NodeCipherReference: Final = 'CipherReference' -NodeCipherValue: Final = 'CipherValue' -NodeDataReference: Final = 'DataReference' -NodeDigestMethod: Final = 'DigestMethod' -NodeDigestValue: Final = 'DigestValue' -NodeEncryptedData: Final = 'EncryptedData' -NodeEncryptedKey: Final = 'EncryptedKey' -NodeEncryptionMethod: Final = 'EncryptionMethod' -NodeEncryptionProperties: Final = 'EncryptionProperties' -NodeEncryptionProperty: Final = 'EncryptionProperty' -NodeKeyInfo: Final = 'KeyInfo' -NodeKeyName: Final = 'KeyName' -NodeKeyReference: Final = 'KeyReference' -NodeKeyValue: Final = 'KeyValue' -NodeManifest: Final = 'Manifest' -NodeObject: Final = 'Object' -NodeReference: Final = 'Reference' -NodeReferenceList: Final = 'ReferenceList' -NodeSignature: Final = 'Signature' -NodeSignatureMethod: Final = 'SignatureMethod' -NodeSignatureProperties: Final = 'SignatureProperties' -NodeSignatureValue: Final = 'SignatureValue' -NodeSignedInfo: Final = 'SignedInfo' -NodeX509Data: Final = 'X509Data' -Ns: Final = 'http://www.aleksey.com/xmlsec/2002' -NsExcC14N: Final = 'http://www.w3.org/2001/10/xml-exc-c14n#' -NsExcC14NWithComments: Final = 'http://www.w3.org/2001/10/xml-exc-c14n#WithComments' -Soap11Ns: Final = 'http://schemas.xmlsoap.org/soap/envelope/' -Soap12Ns: Final = 'http://www.w3.org/2002/06/soap-envelope' -TransformAes128Cbc: Final = __Transform('aes128-cbc', 'http://www.w3.org/2001/04/xmlenc#aes128-cbc', 16) -TransformAes128Gcm: Final = __Transform('aes128-gcm', 'http://www.w3.org/2009/xmlenc11#aes128-gcm', 16) -TransformAes192Cbc: Final = __Transform('aes192-cbc', 'http://www.w3.org/2001/04/xmlenc#aes192-cbc', 16) -TransformAes192Gcm: Final = __Transform('aes192-gcm', 'http://www.w3.org/2009/xmlenc11#aes192-gcm', 16) -TransformAes256Cbc: Final = __Transform('aes256-cbc', 'http://www.w3.org/2001/04/xmlenc#aes256-cbc', 16) -TransformAes256Gcm: Final = __Transform('aes256-gcm', 'http://www.w3.org/2009/xmlenc11#aes256-gcm', 16) -TransformDes3Cbc: Final = __Transform('tripledes-cbc', 'http://www.w3.org/2001/04/xmlenc#tripledes-cbc', 16) -TransformDsaSha1: Final = __Transform('dsa-sha1', 'http://www.w3.org/2000/09/xmldsig#dsa-sha1', 8) -TransformEcdsaSha1: Final = __Transform('ecdsa-sha1', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha1', 8) -TransformEcdsaSha224: Final = __Transform('ecdsa-sha224', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha224', 8) -TransformEcdsaSha256: Final = __Transform('ecdsa-sha256', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256', 8) -TransformEcdsaSha384: Final = __Transform('ecdsa-sha384', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha384', 8) -TransformEcdsaSha512: Final = __Transform('ecdsa-sha512', 'http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha512', 8) -TransformEnveloped: Final = __Transform('enveloped-signature', 'http://www.w3.org/2000/09/xmldsig#enveloped-signature', 1) -TransformExclC14N: Final = __Transform('exc-c14n', 'http://www.w3.org/2001/10/xml-exc-c14n#', 3) -TransformExclC14NWithComments: Final = __Transform( - 'exc-c14n-with-comments', 'http://www.w3.org/2001/10/xml-exc-c14n#WithComments', 3 -) -TransformHmacMd5: Final = __Transform('hmac-md5', 'http://www.w3.org/2001/04/xmldsig-more#hmac-md5', 8) -TransformHmacRipemd160: Final = __Transform('hmac-ripemd160', 'http://www.w3.org/2001/04/xmldsig-more#hmac-ripemd160', 8) -TransformHmacSha1: Final = __Transform('hmac-sha1', 'http://www.w3.org/2000/09/xmldsig#hmac-sha1', 8) -TransformHmacSha224: Final = __Transform('hmac-sha224', 'http://www.w3.org/2001/04/xmldsig-more#hmac-sha224', 8) -TransformHmacSha256: Final = __Transform('hmac-sha256', 'http://www.w3.org/2001/04/xmldsig-more#hmac-sha256', 8) -TransformHmacSha384: Final = __Transform('hmac-sha384', 'http://www.w3.org/2001/04/xmldsig-more#hmac-sha384', 8) -TransformHmacSha512: Final = __Transform('hmac-sha512', 'http://www.w3.org/2001/04/xmldsig-more#hmac-sha512', 8) -TransformInclC14N: Final = __Transform('c14n', 'http://www.w3.org/TR/2001/REC-xml-c14n-20010315', 3) -TransformInclC14N11: Final = __Transform('c14n11', 'http://www.w3.org/2006/12/xml-c14n11', 3) -TransformInclC14N11WithComments: Final = __Transform( - 'c14n11-with-comments', 'http://www.w3.org/2006/12/xml-c14n11#WithComments', 3 -) -TransformInclC14NWithComments: Final = __Transform( - 'c14n-with-comments', - 'http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments', - 3, -) -TransformKWAes128: Final = __Transform('kw-aes128', 'http://www.w3.org/2001/04/xmlenc#kw-aes128', 16) -TransformKWAes192: Final = __Transform('kw-aes192', 'http://www.w3.org/2001/04/xmlenc#kw-aes192', 16) -TransformKWAes256: Final = __Transform('kw-aes256', 'http://www.w3.org/2001/04/xmlenc#kw-aes256', 16) -TransformKWDes3: Final = __Transform('kw-tripledes', 'http://www.w3.org/2001/04/xmlenc#kw-tripledes', 16) -TransformMd5: Final = __Transform('md5', 'http://www.w3.org/2001/04/xmldsig-more#md5', 4) -TransformRemoveXmlTagsC14N: Final = __Transform('remove-xml-tags-transform', None, 3) -TransformRipemd160: Final = __Transform('ripemd160', 'http://www.w3.org/2001/04/xmlenc#ripemd160', 4) -TransformRsaMd5: Final = __Transform('rsa-md5', 'http://www.w3.org/2001/04/xmldsig-more#rsa-md5', 8) -TransformRsaOaep: Final = __Transform('rsa-oaep-mgf1p', 'http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p', 16) -TransformRsaPkcs1: Final = __Transform('rsa-1_5', 'http://www.w3.org/2001/04/xmlenc#rsa-1_5', 16) -TransformRsaRipemd160: Final = __Transform('rsa-ripemd160', 'http://www.w3.org/2001/04/xmldsig-more#rsa-ripemd160', 8) -TransformRsaSha1: Final = __Transform('rsa-sha1', 'http://www.w3.org/2000/09/xmldsig#rsa-sha1', 8) -TransformRsaSha224: Final = __Transform('rsa-sha224', 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha224', 8) -TransformRsaSha256: Final = __Transform('rsa-sha256', 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256', 8) -TransformRsaSha384: Final = __Transform('rsa-sha384', 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha384', 8) -TransformRsaSha512: Final = __Transform('rsa-sha512', 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha512', 8) -TransformSha1: Final = __Transform('sha1', 'http://www.w3.org/2000/09/xmldsig#sha1', 4) -TransformSha224: Final = __Transform('sha224', 'http://www.w3.org/2001/04/xmldsig-more#sha224', 4) -TransformSha256: Final = __Transform('sha256', 'http://www.w3.org/2001/04/xmlenc#sha256', 4) -TransformSha384: Final = __Transform('sha384', 'http://www.w3.org/2001/04/xmldsig-more#sha384', 4) -TransformSha512: Final = __Transform('sha512', 'http://www.w3.org/2001/04/xmlenc#sha512', 4) -TransformUsageAny: Final = 65535 -TransformUsageC14NMethod: Final = 2 -TransformUsageDSigTransform: Final = 1 -TransformUsageDigestMethod: Final = 4 -TransformUsageEncryptionMethod: Final = 16 -TransformUsageSignatureMethod: Final = 8 -TransformUsageUnknown: Final = 0 -TransformVisa3DHack: Final = __Transform('Visa3DHackTransform', None, 1) -TransformXPath: Final = __Transform('xpath', 'http://www.w3.org/TR/1999/REC-xpath-19991116', 1) -TransformXPath2: Final = __Transform('xpath2', 'http://www.w3.org/2002/06/xmldsig-filter2', 1) -TransformXPointer: Final = __Transform('xpointer', 'http://www.w3.org/2001/04/xmldsig-more/xptr', 1) -TransformXslt: Final = __Transform('xslt', 'http://www.w3.org/TR/1999/REC-xslt-19991116', 1) -TypeEncContent: Final = 'http://www.w3.org/2001/04/xmlenc#Content' -TypeEncElement: Final = 'http://www.w3.org/2001/04/xmlenc#Element' -XPath2Ns: Final = 'http://www.w3.org/2002/06/xmldsig-filter2' -XPathNs: Final = 'http://www.w3.org/TR/1999/REC-xpath-19991116' -XPointerNs: Final = 'http://www.w3.org/2001/04/xmldsig-more/xptr' +DSigNs: Final[str] +EncNs: Final[str] +KeyDataAes: Final[__KeyData] +KeyDataDes: Final[__KeyData] +KeyDataDsa: Final[__KeyData] +KeyDataEcdsa: Final[__KeyData] +KeyDataEncryptedKey: Final[__KeyData] +KeyDataFormatBinary: Final[int] +KeyDataFormatCertDer: Final[int] +KeyDataFormatCertPem: Final[int] +KeyDataFormatDer: Final[int] +KeyDataFormatPem: Final[int] +KeyDataFormatPkcs12: Final[int] +KeyDataFormatPkcs8Der: Final[int] +KeyDataFormatPkcs8Pem: Final[int] +KeyDataFormatUnknown: Final[int] +KeyDataHmac: Final[__KeyData] +KeyDataName: Final[__KeyData] +KeyDataRawX509Cert: Final[__KeyData] +KeyDataRetrievalMethod: Final[__KeyData] +KeyDataRsa: Final[__KeyData] +KeyDataTypeAny: Final[int] +KeyDataTypeNone: Final[int] +KeyDataTypePermanent: Final[int] +KeyDataTypePrivate: Final[int] +KeyDataTypePublic: Final[int] +KeyDataTypeSession: Final[int] +KeyDataTypeSymmetric: Final[int] +KeyDataTypeTrusted: Final[int] +KeyDataTypeUnknown: Final[int] +KeyDataValue: Final[__KeyData] +KeyDataX509: Final[__KeyData] +NodeCanonicalizationMethod: Final[str] +NodeCipherData: Final[str] +NodeCipherReference: Final[str] +NodeCipherValue: Final[str] +NodeDataReference: Final[str] +NodeDigestMethod: Final[str] +NodeDigestValue: Final[str] +NodeEncryptedData: Final[str] +NodeEncryptedKey: Final[str] +NodeEncryptionMethod: Final[str] +NodeEncryptionProperties: Final[str] +NodeEncryptionProperty: Final[str] +NodeKeyInfo: Final[str] +NodeKeyName: Final[str] +NodeKeyReference: Final[str] +NodeKeyValue: Final[str] +NodeManifest: Final[str] +NodeObject: Final[str] +NodeReference: Final[str] +NodeReferenceList: Final[str] +NodeSignature: Final[str] +NodeSignatureMethod: Final[str] +NodeSignatureProperties: Final[str] +NodeSignatureValue: Final[str] +NodeSignedInfo: Final[str] +NodeX509Data: Final[str] +Ns: Final[str] +NsExcC14N: Final[str] +NsExcC14NWithComments: Final[str] +Soap11Ns: Final[str] +Soap12Ns: Final[str] +TransformAes128Cbc: Final[__Transform] +TransformAes128Gcm: Final[__Transform] +TransformAes192Cbc: Final[__Transform] +TransformAes192Gcm: Final[__Transform] +TransformAes256Cbc: Final[__Transform] +TransformAes256Gcm: Final[__Transform] +TransformDes3Cbc: Final[__Transform] +TransformDsaSha1: Final[__Transform] +TransformEcdsaSha1: Final[__Transform] +TransformEcdsaSha224: Final[__Transform] +TransformEcdsaSha256: Final[__Transform] +TransformEcdsaSha384: Final[__Transform] +TransformEcdsaSha512: Final[__Transform] +TransformEnveloped: Final[__Transform] +TransformExclC14N: Final[__Transform] +TransformExclC14NWithComments: Final[__Transform] +TransformHmacMd5: Final[__Transform] +TransformHmacRipemd160: Final[__Transform] +TransformHmacSha1: Final[__Transform] +TransformHmacSha224: Final[__Transform] +TransformHmacSha256: Final[__Transform] +TransformHmacSha384: Final[__Transform] +TransformHmacSha512: Final[__Transform] +TransformInclC14N: Final[__Transform] +TransformInclC14N11: Final[__Transform] +TransformInclC14N11WithComments: Final[__Transform] +TransformInclC14NWithComments: Final[__Transform] +TransformKWAes128: Final[__Transform] +TransformKWAes192: Final[__Transform] +TransformKWAes256: Final[__Transform] +TransformKWDes3: Final[__Transform] +TransformMd5: Final[__Transform] +TransformRemoveXmlTagsC14N: Final[__Transform] +TransformRipemd160: Final[__Transform] +TransformRsaMd5: Final[__Transform] +TransformRsaOaep: Final[__Transform] +TransformRsaPkcs1: Final[__Transform] +TransformRsaRipemd160: Final[__Transform] +TransformRsaSha1: Final[__Transform] +TransformRsaSha224: Final[__Transform] +TransformRsaSha256: Final[__Transform] +TransformRsaSha384: Final[__Transform] +TransformRsaSha512: Final[__Transform] +TransformSha1: Final[__Transform] +TransformSha224: Final[__Transform] +TransformSha256: Final[__Transform] +TransformSha384: Final[__Transform] +TransformSha512: Final[__Transform] +TransformUsageAny: Final[int] +TransformUsageC14NMethod: Final[int] +TransformUsageDSigTransform: Final[int] +TransformUsageDigestMethod: Final[int] +TransformUsageEncryptionMethod: Final[int] +TransformUsageSignatureMethod: Final[int] +TransformUsageUnknown: Final[int] +TransformVisa3DHack: Final[__Transform] +TransformXPath: Final[__Transform] +TransformXPath2: Final[__Transform] +TransformXPointer: Final[__Transform] +TransformXslt: Final[__Transform] +TypeEncContent: Final[str] +TypeEncElement: Final[str] +XPath2Ns: Final[str] +XPathNs: Final[str] +XPointerNs: Final[str] diff --git a/src/xmlsec/template.pyi b/src/xmlsec/template.pyi index 162fe25d..a5199fc8 100644 --- a/src/xmlsec/template.pyi +++ b/src/xmlsec/template.pyi @@ -1,16 +1,17 @@ -from typing import Any, Optional, Sequence, Union +from collections.abc import Sequence +from typing import Any from lxml.etree import _Element from xmlsec.constants import __Transform as Transform def add_encrypted_key( - node: _Element, method: Transform, id: Optional[str] = None, type: Optional[str] = None, recipient: Optional[str] = None + node: _Element, method: Transform, id: str | None = ..., type: str | None = ..., recipient: str | None = ... ) -> _Element: ... -def add_key_name(node: _Element, name: Optional[str] = ...) -> _Element: ... +def add_key_name(node: _Element, name: str | None = ...) -> _Element: ... def add_key_value(node: _Element) -> _Element: ... def add_reference( - node: _Element, digest_method: Transform, id: Optional[str] = ..., uri: Optional[str] = ..., type: Optional[str] = ... + node: _Element, digest_method: Transform, id: str | None = ..., uri: str | None = ..., type: str | None = ... ) -> _Element: ... def add_transform(node: _Element, transform: Transform) -> Any: ... def add_x509_data(node: _Element) -> _Element: ... @@ -18,20 +19,20 @@ def create(node: _Element, c14n_method: Transform, sign_method: Transform) -> _E def encrypted_data_create( node: _Element, method: Transform, - id: Optional[str] = ..., - type: Optional[str] = ..., - mime_type: Optional[str] = ..., - encoding: Optional[str] = ..., - ns: Optional[str] = ..., + id: str | None = ..., + type: str | None = ..., + mime_type: str | None = ..., + encoding: str | None = ..., + ns: str | None = ..., ) -> _Element: ... def encrypted_data_ensure_cipher_value(node: _Element) -> _Element: ... -def encrypted_data_ensure_key_info(node: _Element, id: Optional[str] = ..., ns: Optional[str] = ...) -> _Element: ... -def ensure_key_info(node: _Element, id: Optional[str] = ...) -> _Element: ... -def transform_add_c14n_inclusive_namespaces(node: _Element, prefixes: Union[str, Sequence[str]]) -> None: ... +def encrypted_data_ensure_key_info(node: _Element, id: str | None = ..., ns: str | None = ...) -> _Element: ... +def ensure_key_info(node: _Element, id: str | None = ...) -> _Element: ... +def transform_add_c14n_inclusive_namespaces(node: _Element, prefixes: str | Sequence[str]) -> None: ... def x509_data_add_certificate(node: _Element) -> _Element: ... def x509_data_add_crl(node: _Element) -> _Element: ... def x509_data_add_issuer_serial(node: _Element) -> _Element: ... def x509_data_add_ski(node: _Element) -> _Element: ... def x509_data_add_subject_name(node: _Element) -> _Element: ... -def x509_issuer_serial_add_issuer_name(node: _Element, name: Optional[str] = ...) -> _Element: ... -def x509_issuer_serial_add_serial_number(node: _Element, serial: Optional[str] = ...) -> _Element: ... +def x509_issuer_serial_add_issuer_name(node: _Element, name: str | None = ...) -> _Element: ... +def x509_issuer_serial_add_serial_number(node: _Element, serial: str | None = ...) -> _Element: ... diff --git a/src/xmlsec/tree.pyi b/src/xmlsec/tree.pyi index 6447fd08..9f96e447 100644 --- a/src/xmlsec/tree.pyi +++ b/src/xmlsec/tree.pyi @@ -1,17 +1,18 @@ -from typing import Optional, overload, Sequence +from collections.abc import Sequence +from typing import overload from lxml.etree import _Element def add_ids(node: _Element, ids: Sequence[str]) -> None: ... @overload -def find_child(parent: _Element, name: str) -> Optional[_Element]: ... +def find_child(parent: _Element, name: str) -> _Element | None: ... @overload -def find_child(parent: _Element, name: str, namespace: str = ...) -> Optional[_Element]: ... +def find_child(parent: _Element, name: str, namespace: str = ...) -> _Element | None: ... @overload -def find_node(node: _Element, name: str) -> Optional[_Element]: ... +def find_node(node: _Element, name: str) -> _Element | None: ... @overload -def find_node(node: _Element, name: str, namespace: str = ...) -> Optional[_Element]: ... +def find_node(node: _Element, name: str, namespace: str = ...) -> _Element | None: ... @overload -def find_parent(node: _Element, name: str) -> Optional[_Element]: ... +def find_parent(node: _Element, name: str) -> _Element | None: ... @overload -def find_parent(node: _Element, name: str, namespace: str = ...) -> Optional[_Element]: ... +def find_parent(node: _Element, name: str, namespace: str = ...) -> _Element | None: ... diff --git a/tests/test_type_stubs.py b/tests/test_type_stubs.py index 3b1c3757..2312c4d9 100644 --- a/tests/test_type_stubs.py +++ b/tests/test_type_stubs.py @@ -1,6 +1,7 @@ """Test type stubs for correctness where possible.""" import os +import pathlib import sys import pytest @@ -10,12 +11,6 @@ black = pytest.importorskip('black') -if sys.version_info >= (3, 4): - from pathlib import Path -else: - from _pytest.pathlib import Path - - constants_stub_header = """ import sys from typing import NamedTuple @@ -27,12 +22,12 @@ class __KeyData(NamedTuple): # __KeyData type - href: str + href: str | None name: str class __Transform(NamedTuple): # __Transform type - href: str + href: str | None name: str usage: int @@ -51,10 +46,11 @@ def gen_constants_stub(): def process_constant(name): """Generate line in stub file for constant name.""" obj = getattr(xmlsec.constants, name) - return '{name}: Final = {obj!r}'.format(name=name, obj=obj) + return '{name}: Final[{type_name}]'.format(name=name, type_name=type(obj).__name__) names = list(sorted(name for name in dir(xmlsec.constants) if not name.startswith('__'))) lines = [process_constant(name) for name in names] + pathlib.Path('constants_stub_gen.pyi').write_text(constants_stub_header + os.linesep.join(lines)) return constants_stub_header + os.linesep.join(lines) @@ -64,8 +60,7 @@ def test_xmlsec_constants_stub(request): Compare it against the existing stub :file:`xmlsec/constants.pyi`. """ - rootdir = Path(str(request.config.rootdir)) - stub = rootdir / 'src' / 'xmlsec' / 'constants.pyi' - mode = black.FileMode(target_versions=[black.TargetVersion.PY38], line_length=130, is_pyi=True, string_normalization=False) + stub = request.config.rootpath / 'src' / 'xmlsec' / 'constants.pyi' + mode = black.FileMode(target_versions={black.TargetVersion.PY39}, line_length=130, is_pyi=True, string_normalization=False) formatted = black.format_file_contents(gen_constants_stub(), fast=False, mode=mode) assert formatted == stub.read_text() From 7d139b39d1cf6774246c2f5be15684a3a5a69c71 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 14:54:32 +0200 Subject: [PATCH 072/211] fix unwanted replacement with sed Signed-off-by: oleg.hoefling --- src/xmlsec/__init__.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 43959222..7874fc13 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -1,12 +1,12 @@ from collections.abc import Callable, Iterable -from typing import Any, AnyStr, IO, typeVar, overload +from typing import Any, AnyStr, IO, TypeVar, overload from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element from xmlsec.constants import __KeyData as KeyData, __Transform as Transform -_E = typeVar('_E', bound=_Element) +_E = TypeVar('_E', bound=_Element) def enable_debug_trace(enabled: bool = ...) -> None: ... def init() -> None: ... From d5f5306470e0008254367b83f4c69c822a68093c Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 14:55:45 +0200 Subject: [PATCH 073/211] drop unused imports in stub generation test Signed-off-by: oleg.hoefling --- tests/test_type_stubs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_type_stubs.py b/tests/test_type_stubs.py index 2312c4d9..af7d53be 100644 --- a/tests/test_type_stubs.py +++ b/tests/test_type_stubs.py @@ -1,8 +1,6 @@ """Test type stubs for correctness where possible.""" import os -import pathlib -import sys import pytest @@ -50,7 +48,6 @@ def process_constant(name): names = list(sorted(name for name in dir(xmlsec.constants) if not name.startswith('__'))) lines = [process_constant(name) for name in names] - pathlib.Path('constants_stub_gen.pyi').write_text(constants_stub_header + os.linesep.join(lines)) return constants_stub_header + os.linesep.join(lines) From ca566bbfb4941796bb3536fb06654a420178af91 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 15:10:35 +0200 Subject: [PATCH 074/211] mark constants with no href with special type in stubs Signed-off-by: oleg.hoefling --- src/xmlsec/constants.pyi | 23 ++++++++++++++++------- tests/test_type_stubs.py | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 8e2165d8..3430a027 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -7,11 +7,20 @@ else: from typing_extensions import Final class __KeyData(NamedTuple): # __KeyData type - href: str | None + href: str + name: str + +class __KeyDataNoHref(NamedTuple): # __KeyData type + href: None name: str class __Transform(NamedTuple): # __Transform type - href: str | None + href: str + name: str + usage: int + +class __TransformNoHref(NamedTuple): # __Transform type + href: None name: str usage: int @@ -32,9 +41,9 @@ KeyDataFormatPkcs8Der: Final[int] KeyDataFormatPkcs8Pem: Final[int] KeyDataFormatUnknown: Final[int] KeyDataHmac: Final[__KeyData] -KeyDataName: Final[__KeyData] +KeyDataName: Final[__KeyDataNoHref] KeyDataRawX509Cert: Final[__KeyData] -KeyDataRetrievalMethod: Final[__KeyData] +KeyDataRetrievalMethod: Final[__KeyDataNoHref] KeyDataRsa: Final[__KeyData] KeyDataTypeAny: Final[int] KeyDataTypeNone: Final[int] @@ -45,7 +54,7 @@ KeyDataTypeSession: Final[int] KeyDataTypeSymmetric: Final[int] KeyDataTypeTrusted: Final[int] KeyDataTypeUnknown: Final[int] -KeyDataValue: Final[__KeyData] +KeyDataValue: Final[__KeyDataNoHref] KeyDataX509: Final[__KeyData] NodeCanonicalizationMethod: Final[str] NodeCipherData: Final[str] @@ -110,7 +119,7 @@ TransformKWAes192: Final[__Transform] TransformKWAes256: Final[__Transform] TransformKWDes3: Final[__Transform] TransformMd5: Final[__Transform] -TransformRemoveXmlTagsC14N: Final[__Transform] +TransformRemoveXmlTagsC14N: Final[__TransformNoHref] TransformRipemd160: Final[__Transform] TransformRsaMd5: Final[__Transform] TransformRsaOaep: Final[__Transform] @@ -133,7 +142,7 @@ TransformUsageDigestMethod: Final[int] TransformUsageEncryptionMethod: Final[int] TransformUsageSignatureMethod: Final[int] TransformUsageUnknown: Final[int] -TransformVisa3DHack: Final[__Transform] +TransformVisa3DHack: Final[__TransformNoHref] TransformXPath: Final[__Transform] TransformXPath2: Final[__Transform] TransformXPointer: Final[__Transform] diff --git a/tests/test_type_stubs.py b/tests/test_type_stubs.py index af7d53be..9ed8f1e2 100644 --- a/tests/test_type_stubs.py +++ b/tests/test_type_stubs.py @@ -18,17 +18,23 @@ else: from typing_extensions import Final - class __KeyData(NamedTuple): # __KeyData type - href: str | None + href: str name: str +class __KeyDataNoHref(NamedTuple): # __KeyData type + href: None + name: str class __Transform(NamedTuple): # __Transform type - href: str | None + href: str name: str usage: int +class __TransformNoHref(NamedTuple): # __Transform type + href: None + name: str + usage: int """ @@ -44,7 +50,10 @@ def gen_constants_stub(): def process_constant(name): """Generate line in stub file for constant name.""" obj = getattr(xmlsec.constants, name) - return '{name}: Final[{type_name}]'.format(name=name, type_name=type(obj).__name__) + type_name = type(obj).__name__ + if type_name in ('__KeyData', '__Transform') and obj.href is None: + type_name += 'NoHref' + return '{name}: Final[{type_name}]'.format(name=name, type_name=type_name) names = list(sorted(name for name in dir(xmlsec.constants) if not name.startswith('__'))) lines = [process_constant(name) for name in names] From 20be5766fe8100da7a367b38f6878050b29ca068 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 15:15:27 +0200 Subject: [PATCH 075/211] add explicit export for constants module Signed-off-by: oleg.hoefling --- src/xmlsec/__init__.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 7874fc13..90881894 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -4,6 +4,7 @@ from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element +from xmlsec import constants as constants, tree as tree from xmlsec.constants import __KeyData as KeyData, __Transform as Transform _E = TypeVar('_E', bound=_Element) From e8ec653591ac9db340a6223606420e2a2d173754 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 15:21:26 +0200 Subject: [PATCH 076/211] skip stub generation test on windows Signed-off-by: oleg.hoefling --- .appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index f7b9e16d..9155acc0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -44,7 +44,6 @@ build_script: test: off test_script: - pip install -r requirements-test.txt - - pip install black # for stub generation tests - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist - pytest -v --color=yes --junitxml=unittests.xml - ps: Get-ChildItem dist\*.whl | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } From 4062036663b5b516c65c7be179936877a4420500 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 16:29:02 +0200 Subject: [PATCH 077/211] introduce pre-commit Signed-off-by: oleg.hoefling --- .gitignore | 1 + .pre-commit-config.yaml | 48 ++++++++ doc/Makefile | 2 +- doc/source/conf.py | 23 ++-- mypy.ini | 16 --- pyproject.toml | 21 ++++ setup.py | 58 +++++---- src/xmlsec/__init__.pyi | 14 ++- tests/base.py | 29 ++--- tests/data/deskey.bin | 2 +- tests/data/doc.xml | 2 +- tests/data/enc1-in.xml | 2 +- tests/data/enc1-out.xml | 2 +- tests/data/enc2-out.xml | 2 +- tests/data/rsacert.pem | 8 +- tests/data/sign1-in.xml | 5 +- tests/data/sign1-out.xml | 4 +- tests/data/sign2-in.xml | 4 +- tests/data/sign2-out.xml | 4 +- tests/data/sign3-in.xml | 4 +- tests/data/sign3-out.xml | 4 +- tests/data/sign4-in.xml | 4 +- tests/data/sign4-out.xml | 2 +- xmlsec_extra.py | 98 --------------- xmlsec_setupinfo.py | 258 --------------------------------------- 25 files changed, 163 insertions(+), 454 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 mypy.ini delete mode 100644 xmlsec_extra.py delete mode 100644 xmlsec_setupinfo.py diff --git a/.gitignore b/.gitignore index 669df482..15f47985 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !.appveyor* !.git* !.readthedocs.yaml +!.pre-commit-config.yaml # Python /dist diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..3080b068 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +exclude: ".*.diff" # exclude patches +repos: +- repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + types: [] + files: ^.*.pyi?$ + exclude: ^doc/ +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: no-commit-to-branch + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-ast + - id: check-merge-conflict + - id: check-json + - id: detect-private-key + - id: mixed-line-ending + - id: pretty-format-json + args: [--autofix] +- repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + exclude: ^setup.py$ + additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] +- repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.971 + hooks: + - id: mypy + exclude: (setup.py|tests/.*.py|doc/.*) + types: [] + files: ^.*.pyi?$ + additional_dependencies: [lxml-stubs,types-docutils] +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: rst-backticks diff --git a/doc/Makefile b/doc/Makefile index 119469fb..9b6324b7 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/source/conf.py b/doc/source/conf.py index 8e39a5b6..7ff2b1f8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -4,8 +4,11 @@ import urllib.request import lxml -from docutils.nodes import reference -from packaging.version import parse +from docutils.nodes import Text, reference +from packaging.version import Version, parse +from sphinx.addnodes import pending_xref +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError if sys.version_info >= (3, 8): @@ -23,22 +26,22 @@ master_doc = 'index' project = u'python-xmlsec' -copyright = u'2020, Oleg Hoefling ' +copyright = u'2020, Oleg Hoefling ' # noqa: A001 author = u'Bulat Gaifullin ' release = importlib_metadata.version('xmlsec') -parsed = parse(release) +parsed: Version = parse(release) version = '{}.{}'.format(parsed.major, parsed.minor) language = None -exclude_patterns = [] +exclude_patterns: list[str] = [] pygments_style = 'sphinx' todo_include_todos = False html_theme = 'furo' -html_static_path = [] +html_static_path: list[str] = [] htmlhelp_basename = 'python-xmlsecdoc' -latex_elements = {} +latex_elements: dict[str, str] = {} latex_documents = [ ( master_doc, @@ -72,7 +75,7 @@ lxml_element_cls_doc_uri = 'https://lxml.de/api/lxml.etree._Element-class.html' -def lxml_element_doc_reference(app, env, node, contnode): +def lxml_element_doc_reference(app: Sphinx, env: BuildEnvironment, node: pending_xref, contnode: Text) -> reference: """ Handle a missing reference only if it is a ``lxml.etree._Element`` ref. @@ -83,13 +86,13 @@ def lxml_element_doc_reference(app, env, node, contnode): and node.get('reftarget', None) == 'lxml.etree._Element' and contnode.astext() in ('lxml.etree._Element', '_Element') ): - reftitle = '(in lxml v{})'.format(lxml.__version__) + reftitle = '(in lxml v{})'.format(lxml.__version__) # type: ignore[attr-defined] newnode = reference('', '', internal=False, refuri=lxml_element_cls_doc_uri, reftitle=reftitle) newnode.append(contnode) return newnode -def setup(app): +def setup(app: Sphinx) -> None: # first, check whether the doc URL is still valid if urllib.request.urlopen(lxml_element_cls_doc_uri).getcode() != 200: raise ExtensionError('URL to `lxml.etree._Element` docs is not accesible.') diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index f404c54c..00000000 --- a/mypy.ini +++ /dev/null @@ -1,16 +0,0 @@ -[mypy] -files = src -ignore_missing_imports = False -warn_unused_configs = True -disallow_subclassing_any = True -disallow_any_generics = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_return_any = True -no_implicit_reexport = True diff --git a/pyproject.toml b/pyproject.toml index 2bfc3d21..e28878e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,24 @@ +[tool.mypy] +files = ['src'] +ignore_missing_imports = false +warn_unused_configs = true +disallow_subclassing_any = true +disallow_any_generics = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +disallow_any_unimported = true +strict_optional = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +warn_no_return = true +no_implicit_reexport = true +show_error_codes = true + [tool.black] line_length = 130 skip-string-normalization = true diff --git a/setup.py b/setup.py index 9dbf069d..da3a0ad4 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,8 @@ import tarfile import zipfile from distutils import log -from distutils.version import StrictVersion as Version from distutils.errors import DistutilsError +from distutils.version import StrictVersion as Version from pathlib import Path from urllib.request import urlcleanup, urljoin, urlopen, urlretrieve @@ -59,33 +59,23 @@ def latest_release_from_gnome_org_cache(url, lib_name): def latest_zlib_release(): - return latest_release_from_html( - 'https://zlib.net/fossils', re.compile('zlib-(?P.*).tar.gz') - ) + return latest_release_from_html('https://zlib.net/fossils', re.compile('zlib-(?P.*).tar.gz')) def latest_libiconv_release(): - return latest_release_from_html( - 'https://ftp.gnu.org/pub/gnu/libiconv', re.compile('libiconv-(?P.*).tar.gz') - ) + return latest_release_from_html('https://ftp.gnu.org/pub/gnu/libiconv', re.compile('libiconv-(?P.*).tar.gz')) def latest_libxml2_release(): - return latest_release_from_gnome_org_cache( - 'https://download.gnome.org/sources/libxml2', 'libxml2' - ) + return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxml2', 'libxml2') def latest_libxslt_release(): - return latest_release_from_gnome_org_cache( - 'https://download.gnome.org/sources/libxslt', 'libxslt' - ) + return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxslt', 'libxslt') def latest_xmlsec_release(): - return latest_release_from_html( - 'https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz') - ) + return latest_release_from_html('https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz')) class build_ext(build_ext_orig): @@ -265,7 +255,9 @@ def prepare_static_build_linux(self): self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION unset, downloading latest from {}'.format(url))) else: url = 'https://zlib.net/fossils/zlib-{}.tar.gz'.format(self.zlib_version) - self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION={}, downloading from {}'.format(self.zlib_version, url))) + self.info( + '{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION={}, downloading from {}'.format(self.zlib_version, url)) + ) urlretrieve(url, str(zlib_tar)) # fetch libiconv @@ -278,7 +270,11 @@ def prepare_static_build_linux(self): self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {}'.format(url))) else: url = 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{}.tar.gz'.format(self.libiconv_version) - self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_LIBICONV_VERSION={}, downloading from {}'.format(self.libiconv_version, url))) + self.info( + '{:10}: {}'.format( + 'zlib', 'PYXMLSEC_LIBICONV_VERSION={}, downloading from {}'.format(self.libiconv_version, url) + ) + ) urlretrieve(url, str(libiconv_tar)) # fetch libxml2 @@ -290,8 +286,14 @@ def prepare_static_build_linux(self): self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {}'.format(url))) else: version_prefix, _ = self.libxml2_version.split('.', -1) - url = 'https://download.gnome.org/sources/libxml2/{}/libxml2-{}.tar.xz'.format(version_prefix, self.libxml2_version) - self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION={}, downloading from {}'.format(self.libxml2_version, url))) + url = 'https://download.gnome.org/sources/libxml2/{}/libxml2-{}.tar.xz'.format( + version_prefix, self.libxml2_version + ) + self.info( + '{:10}: {}'.format( + 'libxml2', 'PYXMLSEC_LIBXML2_VERSION={}, downloading from {}'.format(self.libxml2_version, url) + ) + ) libxml2_tar = self.libs_dir / 'libxml2.tar.xz' urlretrieve(url, str(libxml2_tar)) @@ -304,8 +306,14 @@ def prepare_static_build_linux(self): self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {}'.format(url))) else: version_prefix, _ = self.libxslt_version.split('.', -1) - url = 'https://download.gnome.org/sources/libxslt/{}/libxslt-{}.tar.xz'.format(version_prefix, self.libxslt_version) - self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION={}, downloading from {}'.format(self.libxslt_version, url))) + url = 'https://download.gnome.org/sources/libxslt/{}/libxslt-{}.tar.xz'.format( + version_prefix, self.libxslt_version + ) + self.info( + '{:10}: {}'.format( + 'libxslt', 'PYXMLSEC_LIBXSLT_VERSION={}, downloading from {}'.format(self.libxslt_version, url) + ) + ) libxslt_tar = self.libs_dir / 'libxslt.tar.gz' urlretrieve(url, str(libxslt_tar)) @@ -318,7 +326,11 @@ def prepare_static_build_linux(self): self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {}'.format(url))) else: url = 'https://www.aleksey.com/xmlsec/download/xmlsec1-{}.tar.gz'.format(self.xmlsec1_version) - self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION={}, downloading from {}'.format(self.xmlsec1_version, url))) + self.info( + '{:10}: {}'.format( + 'xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION={}, downloading from {}'.format(self.xmlsec1_version, url) + ) + ) xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' urlretrieve(url, str(xmlsec1_tar)) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 90881894..be0bd659 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -1,11 +1,13 @@ from collections.abc import Callable, Iterable -from typing import Any, AnyStr, IO, TypeVar, overload -from _typeshed import GenericPath, Self, StrOrBytesPath +from typing import IO, Any, AnyStr, TypeVar, overload +from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element -from xmlsec import constants as constants, tree as tree -from xmlsec.constants import __KeyData as KeyData, __Transform as Transform +from xmlsec import constants as constants +from xmlsec import tree as tree +from xmlsec.constants import __KeyData as KeyData +from xmlsec.constants import __Transform as Transform _E = TypeVar('_E', bound=_Element) @@ -44,12 +46,12 @@ class Key: @classmethod def from_binary_file(cls: type[Self], klass: KeyData, filename: StrOrBytesPath) -> Self: ... @classmethod - def from_file(cls: type[Self], file: GenericPath | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... + def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... - def load_cert_from_file(self, file: GenericPath | IO[AnyStr], format: int) -> None: ... + def load_cert_from_file(self, file: GenericPath[AnyStr] | IO[AnyStr], format: int) -> None: ... def load_cert_from_memory(self, data: AnyStr, format: int) -> None: ... def __copy__(self: Self) -> Self: ... def __deepcopy__(self: Self) -> Self: ... diff --git a/tests/base.py b/tests/base.py index b6b3b56a..06acf413 100644 --- a/tests/base.py +++ b/tests/base.py @@ -7,10 +7,6 @@ import xmlsec -if sys.version_info < (3,): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - - etype = type(etree.Element("test")) ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} @@ -22,7 +18,6 @@ def get_memory_usage(): return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - except ImportError: resource = None @@ -62,7 +57,7 @@ def run(self, result=None): o_count = gc.get_count()[0] m_hits = 0 o_hits = 0 - for i in range(self.iterations): + for _ in range(self.iterations): super(TestMemoryLeaks, self).run(result=result) m_usage_n = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss if m_usage_n > m_usage: @@ -89,16 +84,16 @@ def run(self, result=None): result.addError(self, sys.exc_info()) def path(self, name): - """returns full path for resource""" + """Return full path for resource.""" return os.path.join(self.data_dir, name) def load(self, name): - """loads resource by name""" + """Load resource by name.""" with open(self.path(name), "rb") as stream: return stream.read() def load_xml(self, name, xpath=None): - """returns xml.etree""" + """Return xml.etree.""" with open(self.path(name)) as f: root = etree.parse(f).getroot() if xpath is None: @@ -108,25 +103,25 @@ def load_xml(self, name, xpath=None): def dump(self, root): print(etree.tostring(root)) - def assertXmlEqual(self, first, second, msg=None): - """Checks equality of etree.roots""" + def assertXmlEqual(self, first, second, msg=None): # noqa: N802 + """Check equality of etree.roots.""" msg = msg or '' if first.tag != second.tag: - self.fail('Tags do not match: %s and %s. %s' % (first.tag, second.tag, msg)) + self.fail('Tags do not match: {} and {}. {}'.format(first.tag, second.tag, msg)) for name, value in first.attrib.items(): if second.attrib.get(name) != value: - self.fail('Attributes do not match: %s=%r, %s=%r. %s' % (name, value, name, second.attrib.get(name), msg)) + self.fail('Attributes do not match: {}={!r}, {}={!r}. {}'.format(name, value, name, second.attrib.get(name), msg)) for name in second.attrib.keys(): if name not in first.attrib: - self.fail('x2 has an attribute x1 is missing: %s. %s' % (name, msg)) + self.fail('x2 has an attribute x1 is missing: {}. {}'.format(name, msg)) if not xml_text_compare(first.text, second.text): - self.fail('text: %r != %r. %s' % (first.text, second.text, msg)) + self.fail('text: {!r} != {!r}. {}'.format(first.text, second.text, msg)) if not xml_text_compare(first.tail, second.tail): - self.fail('tail: %r != %r. %s' % (first.tail, second.tail, msg)) + self.fail('tail: {!r} != {!r}. {}'.format(first.tail, second.tail, msg)) cl1 = sorted(first.getchildren(), key=lambda x: x.tag) cl2 = sorted(second.getchildren(), key=lambda x: x.tag) if len(cl1) != len(cl2): - self.fail('children length differs, %i != %i. %s' % (len(cl1), len(cl2), msg)) + self.fail('children length differs, {} != {}. {}'.format(len(cl1), len(cl2), msg)) i = 0 for c1, c2 in zip(cl1, cl2): i += 1 diff --git a/tests/data/deskey.bin b/tests/data/deskey.bin index 019924a7..73245c3c 100644 --- a/tests/data/deskey.bin +++ b/tests/data/deskey.bin @@ -1 +1 @@ -012345670123456701234567 \ No newline at end of file +012345670123456701234567 diff --git a/tests/data/doc.xml b/tests/data/doc.xml index fd474859..39f8d761 100644 --- a/tests/data/doc.xml +++ b/tests/data/doc.xml @@ -4,4 +4,4 @@ XML Security Library example: Original XML doc file for enc example. --> Hello, World! - \ No newline at end of file + diff --git a/tests/data/enc1-in.xml b/tests/data/enc1-in.xml index fd474859..39f8d761 100644 --- a/tests/data/enc1-in.xml +++ b/tests/data/enc1-in.xml @@ -4,4 +4,4 @@ XML Security Library example: Original XML doc file for enc example. --> Hello, World! - \ No newline at end of file + diff --git a/tests/data/enc1-out.xml b/tests/data/enc1-out.xml index 9499453b..ab0b1a6c 100644 --- a/tests/data/enc1-out.xml +++ b/tests/data/enc1-out.xml @@ -19,4 +19,4 @@ DY/U86tTpTn95NwHD10SLyrL6rpXdbEuoIQHhWLwV9uQxnJA/Pn1KZ+xXK/fePfP 2pb5Mxd0f+AW56Cs3MfQ9HJkUVeliSi1hVCNCVHTKeMyC2VL6lPhQ9+L01aSeTSY - \ No newline at end of file + diff --git a/tests/data/enc2-out.xml b/tests/data/enc2-out.xml index 6556248e..4b3b5c34 100644 --- a/tests/data/enc2-out.xml +++ b/tests/data/enc2-out.xml @@ -19,4 +19,4 @@ CTBwsOXCAEJYXPkTrnB3qQ== 4m5BRKEswOe8JISY7NrPGLBYv7Ay5pBV+nG6it51gz0= - \ No newline at end of file + diff --git a/tests/data/rsacert.pem b/tests/data/rsacert.pem index 02489a43..e8a68228 100644 --- a/tests/data/rsacert.pem +++ b/tests/data/rsacert.pem @@ -32,13 +32,13 @@ Certificate: 65:c3 Exponent: 65537 (0x10001) X509v3 extensions: - X509v3 Basic Constraints: + X509v3 Basic Constraints: CA:FALSE - Netscape Comment: + Netscape Comment: OpenSSL Generated Certificate - X509v3 Subject Key Identifier: + X509v3 Subject Key Identifier: 24:84:2C:F2:D4:59:20:62:8B:2E:5C:86:90:A3:AA:30:BA:27:1A:9C - X509v3 Authority Key Identifier: + X509v3 Authority Key Identifier: keyid:B4:B9:EF:9A:E6:97:0E:68:65:1E:98:CE:FA:55:0D:89:06:DB:4C:7C DirName:/C=US/ST=California/L=Sunnyvale/O=XML Security Library (http://www.aleksey.com/xmlsec)/OU=Root Certificate/CN=Aleksey Sanin/emailAddress=xmlsec@aleksey.com serial:00 diff --git a/tests/data/sign1-in.xml b/tests/data/sign1-in.xml index ac71a949..0a0cd442 100644 --- a/tests/data/sign1-in.xml +++ b/tests/data/sign1-in.xml @@ -1,6 +1,6 @@ - @@ -24,4 +24,3 @@ XML Security Library example: Simple signature template file for sign1 example. - diff --git a/tests/data/sign1-out.xml b/tests/data/sign1-out.xml index 04d8fed0..f46ac1f4 100644 --- a/tests/data/sign1-out.xml +++ b/tests/data/sign1-out.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign2-in.xml b/tests/data/sign2-in.xml index 5d9fb352..2f2592f2 100644 --- a/tests/data/sign2-in.xml +++ b/tests/data/sign2-in.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign2-out.xml b/tests/data/sign2-out.xml index b37cad94..b5782d6c 100644 --- a/tests/data/sign2-out.xml +++ b/tests/data/sign2-out.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign3-in.xml b/tests/data/sign3-in.xml index f75da16a..96260b8f 100644 --- a/tests/data/sign3-in.xml +++ b/tests/data/sign3-in.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign3-out.xml b/tests/data/sign3-out.xml index 847e1af2..b7bf15c3 100644 --- a/tests/data/sign3-out.xml +++ b/tests/data/sign3-out.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign4-in.xml b/tests/data/sign4-in.xml index cc00479b..d49fc3ae 100644 --- a/tests/data/sign4-in.xml +++ b/tests/data/sign4-in.xml @@ -1,6 +1,6 @@ - diff --git a/tests/data/sign4-out.xml b/tests/data/sign4-out.xml index a6fecb44..bdd1014e 100644 --- a/tests/data/sign4-out.xml +++ b/tests/data/sign4-out.xml @@ -52,4 +52,4 @@ ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL NJ2D - \ No newline at end of file + diff --git a/xmlsec_extra.py b/xmlsec_extra.py deleted file mode 100644 index 9bf6c86a..00000000 --- a/xmlsec_extra.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import sys - -try: - from urlparse import urljoin - from urllib import urlretrieve, urlcleanup -except ImportError: - from urllib.parse import urljoin - from urllib.request import urlretrieve, urlcleanup - - -# use pre-built libraries on Windows -def get_prebuilt_libs(download_dir, static_include_dirs, static_library_dirs): - assert sys.platform.startswith('win') - libs = download_and_extract_windows_binaries(download_dir) - for ln, path in libs.items(): - if ln == 'xmlsec1': - i = os.path.join(path, 'include', 'xmlsec1') - else: - i = os.path.join(path, 'include') - - l = os.path.join(path, 'lib') - assert os.path.exists(i), 'does not exist: %s' % i - assert os.path.exists(l), 'does not exist: %s' % l - static_include_dirs.append(i) - static_library_dirs.append(l) - - -def download_and_extract_windows_binaries(destdir): - url = "https://github.com/bgaifullin/libxml2-win-binaries/releases/download/v2018.08/" - if sys.version_info < (3, 5): - if sys.maxsize > 2147483647: - suffix = "vs2008.win64" - else: - suffix = "vs2008.win32" - else: - if sys.maxsize > 2147483647: - suffix = "win64" - else: - suffix = "win32" - - libs = { - 'libxml2': 'libxml2-2.9.4.{}.zip'.format(suffix), - 'libxslt': 'libxslt-1.1.29.{}.zip'.format(suffix), - 'zlib': 'zlib-1.2.8.{}.zip'.format(suffix), - 'iconv': 'iconv-1.14.{}.zip'.format(suffix), - 'openssl': 'openssl-1.0.1.{}.zip'.format(suffix), - 'xmlsec': 'xmlsec-1.2.24.{}.zip'.format(suffix), - } - - if not os.path.exists(destdir): - os.makedirs(destdir) - - for ln, fn in libs.items(): - srcfile = urljoin(url, fn) - destfile = os.path.join(destdir, fn) - if os.path.exists(destfile + ".keep"): - print('Using local copy of "{}"'.format(srcfile)) - else: - print('Retrieving "%s" to "%s"' % (srcfile, destfile)) - urlcleanup() # work around FTP bug 27973 in Py2.7.12+ - urlretrieve(srcfile, destfile) - - libs[ln] = unpack_zipfile(destfile, destdir) - - return libs - - -def find_top_dir_of_zipfile(zipfile): - topdir = None - files = [f.filename for f in zipfile.filelist] - dirs = [d for d in files if d.endswith('/')] - if dirs: - dirs.sort(key=len) - topdir = dirs[0] - topdir = topdir[:topdir.index("/")+1] - for path in files: - if not path.startswith(topdir): - topdir = None - break - assert topdir, ( - "cannot determine single top-level directory in zip file %s" % - zipfile.filename) - return topdir.rstrip('/') - - -def unpack_zipfile(zipfn, destdir): - assert zipfn.endswith('.zip') - import zipfile - print('Unpacking %s into %s' % (os.path.basename(zipfn), destdir)) - f = zipfile.ZipFile(zipfn) - try: - extracted_dir = os.path.join(destdir, find_top_dir_of_zipfile(f)) - f.extractall(path=destdir) - finally: - f.close() - assert os.path.exists(extracted_dir), 'missing: %s' % extracted_dir - return extracted_dir diff --git a/xmlsec_setupinfo.py b/xmlsec_setupinfo.py deleted file mode 100644 index bcfbd321..00000000 --- a/xmlsec_setupinfo.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import print_function - -import glob -import os -import pkg_resources -import sys - -from distutils.errors import DistutilsOptionError - - -WIN32 = sys.platform.lower().startswith('win') - -__MODULE_NAME = "xmlsec" -__MODULE_VERSION = None -__MODULE_DESCRIPTION = "Python bindings for the XML Security Library" -__MODULE_REQUIREMENTS = None -__XMLSEC_CONFIG = None - - -def name(): - return __MODULE_NAME - - -def version(): - global __MODULE_VERSION - if __MODULE_VERSION is None: - with open(os.path.join(get_base_dir(), 'version.txt')) as f: - __MODULE_VERSION = f.read().strip() - return __MODULE_VERSION - - -def description(): - return __MODULE_DESCRIPTION - - -def sources(): - return sorted(glob.glob(os.path.join(get_base_dir(), "src", "*.c"))) - - -def define_macros(): - macros = [ - ("MODULE_NAME", __MODULE_NAME), - ("MODULE_VERSION", version()), - ] - if OPTION_ENABLE_DEBUG: - macros.append(("PYXMLSEC_ENABLE_DEBUG", "1")) - - macros.extend(xmlsec_config()['define_macros']) - - return macros - - -def cflags(): - options = [] - if WIN32: - options.append("/Zi") - else: - options.append("-g") - options.append("-std=c99") - options.append("-fno-strict-aliasing") - options.append("-Wno-error=declaration-after-statement") - options.append("-Werror=implicit-function-declaration") - - if OPTION_ENABLE_DEBUG: - options.append("-Wall") - options.append("-O0") - else: - options.append("-Os") - - return options - - -def include_dirs(): - import lxml - - dirs = xmlsec_config()['include_dirs'] - dirs.extend(lxml.get_include()) - return dirs - - -def libraries(): - return xmlsec_config()['libraries'] - - -def library_dirs(): - return xmlsec_config()['library_dirs'] - - -def dev_status(): - _version = version() - if 'a' in _version: - return 'Development Status :: 3 - Alpha' - elif 'b' in _version or 'c' in _version: - return 'Development Status :: 4 - Beta' - else: - return 'Development Status :: 5 - Production/Stable' - - -def requirements(): - global __MODULE_REQUIREMENTS - if __MODULE_REQUIREMENTS is None: - with open(os.path.join(get_base_dir(), "requirements.txt")) as f: - __MODULE_REQUIREMENTS = [str(req) for req in pkg_resources.parse_requirements(f)] - return __MODULE_REQUIREMENTS - - -def xmlsec_config(): - global __XMLSEC_CONFIG - - if __XMLSEC_CONFIG is None: - __XMLSEC_CONFIG = load_xmlsec1_config() - - return __XMLSEC_CONFIG - - -def load_xmlsec1_config(): - config = None - - if WIN32: - import xmlsec_extra - - config = { - 'define_macros': [ - ('XMLSEC_CRYPTO', '\\"openssl\\"'), - ('__XMLSEC_FUNCTION__', '__FUNCTION__'), - ('XMLSEC_NO_GOST', '1'), - ('XMLSEC_NO_XKMS', '1'), - ('XMLSEC_NO_CRYPTO_DYNAMIC_LOADING', '1'), - ('XMLSEC_CRYPTO_OPENSSL', '1'), - ('UNICODE', '1'), - ('_UNICODE', '1'), - ('LIBXML_ICONV_ENABLED', 1), - ('LIBXML_STATIC', '1'), - ('LIBXSLT_STATIC', '1'), - ('XMLSEC_STATIC', '1'), - ('inline', '__inline'), - ], - 'libraries': [ - 'libxmlsec_a', - 'libxmlsec-openssl_a', - 'libeay32', - 'iconv_a', - 'libxslt_a', - 'libexslt_a', - 'libxml2_a', - 'zlib', - 'WS2_32', - 'Advapi32', - 'User32', - 'Gdi32', - 'Crypt32', - ], - 'include_dirs': [], - 'library_dirs': [], - } - - xmlsec_extra.get_prebuilt_libs( - OPTION_DOWNLOAD_DIR, config['include_dirs'], config['library_dirs'] - ) - else: - import pkgconfig - - try: - config = pkgconfig.parse('xmlsec1') - except EnvironmentError: - pass - - if config is None or not config.get('libraries'): - fatal_xmlsec1_error() - - # make sure that all options are list - for x in ('libraries', 'include_dirs', 'library_dirs'): - config[x] = list(config.get(x) or []) - - # fix macros, ensure that macros is list - macros = list(config.get('define_macros', [])) - for i, v in enumerate(macros): - if v[0] == 'XMLSEC_CRYPTO' and not (v[1].startswith('"') and v[1].endswith('"')): - macros[i] = ('XMLSEC_CRYPTO', '"{0}"'.format(v[1])) - break - config['define_macros'] = macros - return config - - -def fatal_xmlsec1_error(): - print('*********************************************************************************') - print('Could not find xmlsec1 config. Are libxmlsec1-dev and pkg-config installed?') - if sys.platform in ('darwin',): - print('Perhaps try: xcode-select --install') - print('*********************************************************************************') - sys.exit(1) - - -def get_base_dir(): - return os.path.abspath(os.path.dirname(sys.argv[0])) - - -if sys.version_info[0] >= 3: - _system_encoding = sys.getdefaultencoding() - if _system_encoding is None: - _system_encoding = "iso-8859-1" - - def decode_input(data): - if isinstance(data, str): - return data - return data.decode(_system_encoding) -else: - def decode_input(data): - return data - - -def env_var(n): - value = os.getenv(n) - if value: - value = decode_input(value) - if sys.platform == 'win32' and ';' in value: - return value.split(';') - else: - return value.split() - else: - return [] - - -def env_var_name(n): - return "PYXMLSEC_" + n.upper().replace('-', '_') - - -# Option handling: - -def has_option(n): - try: - sys.argv.remove('--%s' % n) - return True - except ValueError: - pass - # allow passing all cmd line options also as environment variables - env_val = os.getenv(env_var_name(n), 'false').lower() - return env_val in ("true", "1") - - -def option_value(n, default=None): - for index, option in enumerate(sys.argv): - if option == '--' + n: - if index+1 >= len(sys.argv): - raise DistutilsOptionError( - 'The option %s requires a value' % option) - value = sys.argv[index+1] - sys.argv[index:index+2] = [] - return value - if option.startswith('--' + n + '='): - value = option[len(n)+3:] - sys.argv[index:index+1] = [] - return value - return os.getenv(env_var_name(n), default) - - -OPTION_ENABLE_DEBUG = has_option('enable-debug') -OPTION_DOWNLOAD_DIR = option_value('download-dir', 'build/extra') From e2a369c9e73add4776158829ecccb4143b3120ca Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 16:59:08 +0200 Subject: [PATCH 078/211] fix libxml2/libxslt version parsing in setup script Signed-off-by: oleg.hoefling --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da3a0ad4..9a3c9277 100644 --- a/setup.py +++ b/setup.py @@ -285,7 +285,7 @@ def prepare_static_build_linux(self): url = latest_libxml2_release() self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {}'.format(url))) else: - version_prefix, _ = self.libxml2_version.split('.', -1) + version_prefix, _ = self.libxml2_version.rsplit('.', 1) url = 'https://download.gnome.org/sources/libxml2/{}/libxml2-{}.tar.xz'.format( version_prefix, self.libxml2_version ) @@ -305,7 +305,7 @@ def prepare_static_build_linux(self): url = latest_libxslt_release() self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {}'.format(url))) else: - version_prefix, _ = self.libxslt_version.split('.', -1) + version_prefix, _ = self.libxslt_version.rsplit('.', 1) url = 'https://download.gnome.org/sources/libxslt/{}/libxslt-{}.tar.xz'.format( version_prefix, self.libxslt_version ) From 87fb5aa6aae0eb6be81ed46561a299d47aee0df5 Mon Sep 17 00:00:00 2001 From: "oleg.hoefling" Date: Sat, 20 Aug 2022 20:47:38 +0200 Subject: [PATCH 079/211] use python 3.9 in rtd docs builds Signed-off-by: oleg.hoefling --- .readthedocs.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f89031d3..93665c84 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,10 +1,14 @@ version: 2 +build: + os: ubuntu-20.04 + tools: + python: '3.9' + sphinx: configuration: doc/source/conf.py python: - version: 3.8 install: - method: pip path: . From 979be38b9819509591a0c542d126ec57918c612d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sat, 20 Aug 2022 22:39:03 +0200 Subject: [PATCH 080/211] Add CI for Python 3.10, manylinux2, musllinux1 and OpenSUSE Tumbleweed (#225) * add CI for python 3.10 Signed-off-by: oleg.hoefling * skip 3.6 and 3.7 builds for tumbleweed since python versions not available in repo Signed-off-by: oleg.hoefling * build for manylinux2_24 soabi Signed-off-by: oleg.hoefling * try building for musllinux1_1 soabi Signed-off-by: oleg.hoefling * fix wheel selection for auditwheel repair Signed-off-by: oleg.hoefling * use plain parametrization in constants tests instead of hypothesis Signed-off-by: oleg.hoefling * battle zypper Signed-off-by: oleg.hoefling * battle zypper Signed-off-by: oleg.hoefling * battle zypper Signed-off-by: oleg.hoefling * battle zypper Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- .appveyor.yml | 4 +++ .github/workflows/macosx.yml | 2 +- .../{manylinux2010.yml => manylinux.yml} | 23 ++++++++++----- .github/workflows/opensuse-tumbleweed.yml | 29 +++++++++++++++++++ .github/workflows/sdist.yml | 4 +-- requirements-test.txt | 1 - tests/test_constants.py | 10 +++---- 7 files changed, 57 insertions(+), 16 deletions(-) rename .github/workflows/{manylinux2010.yml => manylinux.yml} (75%) create mode 100644 .github/workflows/opensuse-tumbleweed.yml diff --git a/.appveyor.yml b/.appveyor.yml index 9155acc0..de17f8ba 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,6 +12,10 @@ environment: python_version: 3.9.13 - python: 39-x64 python_version: 3.9.13 + - python: 310 + python_version: 3.10.6 + - python: 310-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index ef72fb78..4db5e306 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux2010.yml b/.github/workflows/manylinux.yml similarity index 75% rename from .github/workflows/manylinux2010.yml rename to .github/workflows/manylinux.yml index d1769b74..867c17ba 100644 --- a/.github/workflows/manylinux2010.yml +++ b/.github/workflows/manylinux.yml @@ -1,12 +1,21 @@ -name: manylinux2010 +name: manylinux on: [push, pull_request] jobs: - manylinux2010_x86_64: + pep513: runs-on: ubuntu-latest - container: quay.io/pypa/manylinux2010_x86_64 strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + image: + - manylinux2010_x86_64 + - manylinux_2_24_x86_64 + - musllinux_1_1_x86_64 + exclude: + - image: manylinux2010_x86_64 + python-abi: cp310-cp310 + - image: manylinux2010_i686 + python-abi: cp310-cp310 + container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 - name: Install build dependencies @@ -23,13 +32,13 @@ jobs: PYXMLSEC_STATIC_DEPS: true run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build - - name: Label manylinux2010_x86_64 wheel + - name: Label manylinux wheel run: | ls -la dist/ auditwheel show dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl auditwheel repair dist/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-linux_x86_64.whl - ls -l wheelhouse/ - auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-manylinux_2_12_x86_64.manylinux2010_x86_64.whl + ls -la wheelhouse/ + auditwheel show wheelhouse/xmlsec-${{ env.PKGVER }}-${{ matrix.python-abi }}-*${{ matrix.image }}*.whl - name: Install test dependencies run: | /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade -r requirements-test.txt diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml new file mode 100644 index 00000000..d8bb8113 --- /dev/null +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -0,0 +1,29 @@ +name: opensuse-tumbleweed +on: [push, pull_request] +jobs: + tumbleweed: + runs-on: ubuntu-latest + container: opensuse/tumbleweed + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v1 + - name: Install build dependencies + run: | + zypper -n install -t pattern devel_basis + PKGVER_NO_DOT=$(tr -d '.' <<< ${{ matrix.python-version }}) + zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${PKGVER_NO_DOT}-pip + python${{ matrix.python-version }} -m venv .venv + .venv/bin/python -m pip install --upgrade pip setuptools wheel + - name: Build linux_x86_64 wheel + run: | + .venv/bin/python setup.py bdist_wheel + rm -rf build/ + - name: Install test dependencies + run: | + .venv/bin/python -m pip install --upgrade -r requirements-test.txt + .venv/bin/python -m pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + - name: Run tests + run: | + .venv/bin/python -m pytest -v --color=yes diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 28fe79fe..fb13377a 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/requirements-test.txt b/requirements-test.txt index a2a90bbc..eb543402 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,3 @@ -r requirements.txt pytest>=4.6.9 -hypothesis lxml-stubs diff --git a/tests/test_constants.py b/tests/test_constants.py index 689edce6..f79d19f0 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,6 +1,6 @@ """Test constants from :mod:`xmlsec.constants` module.""" -from hypothesis import given, strategies +import pytest import xmlsec @@ -18,25 +18,25 @@ def _constants(typename): ) -@given(transform=strategies.sampled_from(_constants('__Transform'))) +@pytest.mark.parametrize('transform', _constants('__Transform'), ids=repr) def test_transform_str(transform): """Test string representation of ``xmlsec.constants.__Transform``.""" assert str(transform) == '{}, {}'.format(transform.name, transform.href) -@given(transform=strategies.sampled_from(_constants('__Transform'))) +@pytest.mark.parametrize('transform', _constants('__Transform'), ids=repr) def test_transform_repr(transform): """Test raw string representation of ``xmlsec.constants.__Transform``.""" assert repr(transform) == '__Transform({!r}, {!r}, {})'.format(transform.name, transform.href, transform.usage) -@given(keydata=strategies.sampled_from(_constants('__KeyData'))) +@pytest.mark.parametrize('keydata', _constants('__KeyData'), ids=repr) def test_keydata_str(keydata): """Test string representation of ``xmlsec.constants.__KeyData``.""" assert str(keydata) == '{}, {}'.format(keydata.name, keydata.href) -@given(keydata=strategies.sampled_from(_constants('__KeyData'))) +@pytest.mark.parametrize('keydata', _constants('__KeyData'), ids=repr) def test_keydata_repr(keydata): """Test raw string representation of ``xmlsec.constants.__KeyData``.""" assert repr(keydata) == '__KeyData({!r}, {!r})'.format(keydata.name, keydata.href) From b483b644b6033e539f303509f5ee85bebf178a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sat, 20 Aug 2022 22:54:29 +0200 Subject: [PATCH 081/211] update gh actions badges (#226) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- README.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 1543f821..b2b3ab11 100644 --- a/README.rst +++ b/README.rst @@ -7,12 +7,14 @@ python-xmlsec :target: https://travis-ci.org/mehcode/python-xmlsec .. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor :target: https://ci.appveyor.com/project/hoefling/xmlsec -.. image:: https://github.com/mehcode/python-xmlsec/workflows/manylinux2010/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22manylinux2010%22 -.. image:: https://github.com/mehcode/python-xmlsec/workflows/MacOS/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22MacOS%22 -.. image:: https://github.com/mehcode/python-xmlsec/workflows/linuxbrew/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions?query=workflow%3A%22linuxbrew%22 +.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml/badge.svg + :target: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml +.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/macosx.yml/badge.svg + :target: https://github.com/mehcode/python-xmlsec/actions/workflows/macosx.yml +.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml/badge.svg + :target: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml +.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml/badge.svg + :target: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml .. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg :target: https://codecov.io/gh/mehcode/python-xmlsec .. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs From 21ce91648b8e09dd9d5c60acab01e849196ad699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 00:18:29 +0200 Subject: [PATCH 082/211] allow building docs with python 3.8 (#227) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- doc/source/conf.py | 18 ++++++++++-------- doc/source/index.rst | 3 --- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 7ff2b1f8..01c2b895 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- -import sys +from __future__ import annotations + import urllib.request +import importlib.metadata import lxml from docutils.nodes import Text, reference @@ -11,11 +13,6 @@ from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError -if sys.version_info >= (3, 8): - from importlib import metadata as importlib_metadata -else: - import importlib_metadata - extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] @@ -28,11 +25,10 @@ project = u'python-xmlsec' copyright = u'2020, Oleg Hoefling ' # noqa: A001 author = u'Bulat Gaifullin ' -release = importlib_metadata.version('xmlsec') +release = importlib.metadata.version('xmlsec') parsed: Version = parse(release) version = '{}.{}'.format(parsed.major, parsed.minor) -language = None exclude_patterns: list[str] = [] pygments_style = 'sphinx' todo_include_todos = False @@ -69,6 +65,12 @@ autodoc_member_order = 'groupwise' autodoc_docstring_signature = True + +rst_prolog = ''' +.. role:: xml(code) + :language: xml +''' + # LXML crossref'ing stuff: # LXML doesn't have an intersphinx docs, # so we link to lxml.etree._Element explicitly diff --git a/doc/source/index.rst b/doc/source/index.rst index 5cc758b9..e08f47d9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,9 +3,6 @@ You can adapt this file completely to your liking, but it should at least contain the root ``toctree`` directive. -.. role:: xml(code) - :language: xml - Welcome to python-xmlsec's documentation! ========================================= From 00759a33f6cbc2aa62536a1898d8107f88a49e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 00:20:05 +0200 Subject: [PATCH 083/211] add proper param types in register_callbacks (#228) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- src/main.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.c b/src/main.c index 41f5e5fc..ffcae14f 100644 --- a/src/main.c +++ b/src/main.c @@ -275,18 +275,23 @@ static PyObject* PyXmlSec_PyIORegisterDefaultCallbacks(PyObject *self) { } static char PyXmlSec_PyIORegisterCallbacks__doc__[] = \ + "register_callbacks(input_match_callback, input_open_callback, input_read_callback, input_close_callback) -> None\n" "Register globally a custom set of IO callbacks with xmlsec.\n\n" - ":param callable input_match_callback: A callable that takes a filename `bytestring` and " + ":param input_match_callback: A callable that takes a filename `bytestring` and " "returns a boolean as to whether the other callbacks in this set can handle that name.\n" - ":param callable input_open_callback: A callable that takes a filename and returns some " + ":type input_match_callback: ~collections.abc.Callable[[bytes], bool]\n" + ":param input_open_callback: A callable that takes a filename and returns some " "context object (e.g. a file object) that the remaining callables in this set will be passed " "during handling.\n" + ":type input_open_callback: ~collections.abc.Callable[[bytes], Any]\n" // FIXME: How do we handle failures in ^^ (e.g. can't find the file)? - ":param callable input_read_callback: A callable that that takes the context object from the " + ":param input_read_callback: A callable that that takes the context object from the " "open callback and a buffer, and should fill the buffer with data (e.g. BytesIO.readinto()). " "xmlsec will call this function several times until there is no more data returned.\n" - ":param callable input_close_callback: A callable that takes the context object from the " + ":type input_read_callback: ~collections.abc.Callable[[Any, memoryview], int]\n" + ":param input_close_callback: A callable that takes the context object from the " "open callback and can do any resource cleanup necessary.\n" + ":type input_close_callback: ~collections.abc.Callable[[Any], None]\n" ; static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = { From 824084311638b7a6d07c0e6fa9414f14e84ee713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Sun, 21 Aug 2022 09:10:09 +0200 Subject: [PATCH 084/211] adjust doc examples to #212 (#229) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- doc/source/examples/encrypt.py | 11 +++++++---- doc/source/examples/sign.py | 3 ++- doc/source/examples/verify.py | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/source/examples/encrypt.py b/doc/source/examples/encrypt.py index f69d4613..98f63b6f 100644 --- a/doc/source/examples/encrypt.py +++ b/doc/source/examples/encrypt.py @@ -2,10 +2,9 @@ import xmlsec -manager = xmlsec.KeysManager() -key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) -manager.add_key(key) -template = etree.parse('enc1-doc.xml').getroot() +with open('enc1-doc.xml') as fp: + template = etree.parse(fp).getroot() + enc_data = xmlsec.template.encrypted_data_create( template, xmlsec.constants.TransformAes128Cbc, @@ -20,6 +19,10 @@ data = template.find('./Data') # Encryption +manager = xmlsec.KeysManager() +key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) +manager.add_key(key) + enc_ctx = xmlsec.EncryptionContext(manager) enc_ctx.key = xmlsec.Key.generate( xmlsec.constants.KeyDataAes, 128, xmlsec.constants.KeyDataTypeSession diff --git a/doc/source/examples/sign.py b/doc/source/examples/sign.py index 4529bc8a..519c13a0 100644 --- a/doc/source/examples/sign.py +++ b/doc/source/examples/sign.py @@ -2,7 +2,8 @@ import xmlsec -template = etree.parse('sign1-tmpl.xml').getroot() +with open('sign1-tmpl.xml') as fp: + template = etree.parse(fp).getroot() signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/verify.py b/doc/source/examples/verify.py index 8629c550..808a53c2 100644 --- a/doc/source/examples/verify.py +++ b/doc/source/examples/verify.py @@ -2,7 +2,9 @@ import xmlsec -template = etree.parse('sign1-res.xml').getroot() +with open('sign1-res.xml') as fp: + template = etree.parse(fp).getroot() + xmlsec.tree.add_ids(template, ["ID"]) signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) # Create a digital signature context (no key manager is needed). From c39eeff382ed119913463832f2f44d89b90dcc6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Wed, 24 Aug 2022 20:26:22 +0200 Subject: [PATCH 085/211] Add integration with pre-commit.ci (#230) * add pre-commit.ci badge Signed-off-by: oleg.hoefling * fix pre-commit warnings Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- .pre-commit-config.yaml | 2 +- README.rst | 5 +-- doc/source/conf.py | 5 +-- doc/source/examples/sign_binary.py | 2 -- doc/source/examples/verify_binary.py | 2 -- doc/source/sphinx-pr-6916.diff | 46 ---------------------------- setup.cfg | 3 ++ tests/base.py | 21 +++---------- tests/conftest.py | 5 +-- tests/test_ds.py | 3 +- tests/test_enc.py | 9 +++--- tests/test_main.py | 4 +-- tests/test_xmlsec.py | 6 ++-- 13 files changed, 26 insertions(+), 87 deletions(-) delete mode 100644 doc/source/sphinx-pr-6916.diff diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3080b068..3c89b250 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,5 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: ".*.diff" # exclude patches repos: - repo: https://github.com/psf/black rev: 22.6.0 @@ -21,6 +20,7 @@ repos: - id: check-merge-conflict - id: check-json - id: detect-private-key + exclude: ^.*/rsakey.pem$ - id: mixed-line-ending - id: pretty-format-json args: [--autofix] diff --git a/README.rst b/README.rst index b2b3ab11..03312504 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,9 @@ python-xmlsec .. image:: https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white :target: https://pypi.python.org/pypi/xmlsec -.. image:: https://img.shields.io/travis/com/mehcode/python-xmlsec/master.svg?logo=travis&logoColor=white&label=Travis%20CI - :target: https://travis-ci.org/mehcode/python-xmlsec +.. image:: https://results.pre-commit.ci/badge/github/xmlsec/python-xmlsec/master.svg + :target: https://results.pre-commit.ci/latest/github/xmlsec/python-xmlsec/master + :alt: pre-commit.ci status .. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor :target: https://ci.appveyor.com/project/hoefling/xmlsec .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml/badge.svg diff --git a/doc/source/conf.py b/doc/source/conf.py index 01c2b895..35329050 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- - from __future__ import annotations -import urllib.request import importlib.metadata +import urllib.request import lxml from docutils.nodes import Text, reference @@ -13,7 +11,6 @@ from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError - extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)} diff --git a/doc/source/examples/sign_binary.py b/doc/source/examples/sign_binary.py index 4e6c0e00..275c6e40 100644 --- a/doc/source/examples/sign_binary.py +++ b/doc/source/examples/sign_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/verify_binary.py b/doc/source/examples/verify_binary.py index 06c2b727..1f888b99 100644 --- a/doc/source/examples/verify_binary.py +++ b/doc/source/examples/verify_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/sphinx-pr-6916.diff b/doc/source/sphinx-pr-6916.diff deleted file mode 100644 index e7040a0f..00000000 --- a/doc/source/sphinx-pr-6916.diff +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py -index bc9bf49a74..4804c89c52 100644 ---- a/sphinx/environment/__init__.py -+++ b/sphinx/environment/__init__.py -@@ -46,6 +46,7 @@ - default_settings = { - 'embed_stylesheet': False, - 'cloak_email_addresses': True, -+ 'syntax_highlight': 'short', - 'pep_base_url': 'https://www.python.org/dev/peps/', - 'pep_references': None, - 'rfc_base_url': 'https://tools.ietf.org/html/', -diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py -index 85eeb43..80f1eea 100644 ---- a/sphinx/writers/html.py -+++ b/sphinx/writers/html.py -@@ -494,8 +494,11 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: -diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py -index 80cedd3..470f559 100644 ---- a/sphinx/writers/html5.py -+++ b/sphinx/writers/html5.py -@@ -446,8 +446,11 @@ class HTML5Translator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: diff --git a/setup.cfg b/setup.cfg index 88d815b0..c090b4e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,5 +18,8 @@ upload_dir = doc/build/html [flake8] per-file-ignores = *.pyi: E301, E302, E305, E501, E701, F401, F822 + doc/source/conf.py: D1 + doc/source/examples/*.py: D1, E501 + tests/*.py: D1 exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* max-line-length = 130 diff --git a/tests/base.py b/tests/base.py index 06acf413..29605ce7 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,5 +1,6 @@ import gc import os +import resource import sys import unittest @@ -12,20 +13,8 @@ ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} -try: - import resource - - def get_memory_usage(): - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - -except ImportError: - resource = None - - def get_memory_usage(): - return 0 - - def get_iterations(): + """Parse iterations amount.""" if sys.platform in ('win32',): return 0 @@ -114,9 +103,9 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 for name in second.attrib.keys(): if name not in first.attrib: self.fail('x2 has an attribute x1 is missing: {}. {}'.format(name, msg)) - if not xml_text_compare(first.text, second.text): + if not _xml_text_compare(first.text, second.text): self.fail('text: {!r} != {!r}. {}'.format(first.text, second.text, msg)) - if not xml_text_compare(first.tail, second.tail): + if not _xml_text_compare(first.tail, second.tail): self.fail('tail: {!r} != {!r}. {}'.format(first.tail, second.tail, msg)) cl1 = sorted(first.getchildren(), key=lambda x: x.tag) cl2 = sorted(second.getchildren(), key=lambda x: x.tag) @@ -128,7 +117,7 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 self.assertXmlEqual(c1, c2) -def xml_text_compare(t1, t2): +def _xml_text_compare(t1, t2): if not t1 and not t2: return True if t1 == '*' or t2 == '*': diff --git a/tests/conftest.py b/tests/conftest.py index b51c4d45..675258c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ def pytest_collection_modifyitems(items): """ - Put the module init test first to implicitly check whether - any subsequent test fails because of module reinitialization. + Put the module init test first. + + This way, we implicitly check whether any subsequent test fails because of module reinitialization. """ def module_init_tests_first(item): diff --git a/tests/test_ds.py b/tests/test_ds.py index 9417fedb..694ad431 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -132,7 +132,6 @@ def test_sign_case3(self): def test_sign_case4(self): """Should sign a file using a dynamically created template, key from PEM and an X509 cert with custom ns.""" - root = self.load_xml("sign4-in.xml") xmlsec.tree.add_ids(root, ["ID"]) elem_id = root.get('ID', None) @@ -249,7 +248,7 @@ def test_verify_case_5(self): self.check_verify(5) def check_verify(self, i): - root = self.load_xml("sign%d-out.xml" % i) + root = self.load_xml("sign{}-out.xml".format(i)) xmlsec.tree.add_ids(root, ["ID"]) sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) diff --git a/tests/test_enc.py b/tests/test_enc.py index 7add848c..63e00169 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -1,4 +1,3 @@ -import os import tempfile from lxml import etree @@ -125,7 +124,7 @@ def test_encrypt_binary(self): encrypted = ctx.encrypt_binary(enc_data, b'test') self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -170,7 +169,7 @@ def test_encrypt_uri(self): encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -219,7 +218,7 @@ def test_decrypt_key(self): self.assertEqual(self.load_xml("enc3-in.xml"), decrypted) def check_decrypt(self, i): - root = self.load_xml('enc%d-out.xml' % i) + root = self.load_xml('enc{}-out.xml'.format(i)) enc_data = xmlsec.tree.find_child(root, consts.NodeEncryptedData, consts.EncNs) self.assertIsNotNone(enc_data) @@ -228,7 +227,7 @@ def check_decrypt(self, i): ctx = xmlsec.EncryptionContext(manager) decrypted = ctx.decrypt(enc_data) self.assertIsNotNone(decrypted) - self.assertEqual(self.load_xml("enc%d-in.xml" % i), root) + self.assertEqual(self.load_xml("enc{}-in.xml".format(i)), root) def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() diff --git a/tests/test_main.py b/tests/test_main.py index 9fb71eaf..3db18582 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -112,7 +112,7 @@ def test_sign_data_not_first_callback(self): def match_cb(filename): nonlocal bad_match_calls bad_match_calls += 1 - False + return False for _ in range(2): self._register_mismatch_callbacks(match_cb) @@ -132,7 +132,7 @@ def test_failed_sign_because_default_callbacks(self): def mismatch_cb(filename): nonlocal mismatch_calls mismatch_calls += 1 - False + return False # NB: These first two sets of callbacks should never get called, # because the default callbacks always match beforehand: diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 32fac69a..303d7f8f 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -5,9 +5,9 @@ class TestModule(base.TestMemoryLeaks): def test_reinitialize_module(self): """ - This doesn't explicitly test anything, but will - be invoked first in the suite, so if the subsequent - tests don't fail, we know that the ``init()``/``shutdown()`` + This test doesn't explicitly verify anything, but will be invoked first in the suite. + + So if the subsequent tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. """ xmlsec.shutdown() From 91592052b671d0ccb34a51ff84534cd684ea1473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Wed, 24 Aug 2022 20:53:17 +0200 Subject: [PATCH 086/211] fix test run on appveyor (#231) Signed-off-by: oleg.hoefling Signed-off-by: oleg.hoefling --- tests/base.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/base.py b/tests/base.py index 29605ce7..48aef81d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,6 +1,5 @@ import gc import os -import resource import sys import unittest @@ -8,26 +7,23 @@ import xmlsec -etype = type(etree.Element("test")) +etype = type(etree.Element('test')) ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} -def get_iterations(): - """Parse iterations amount.""" - if sys.platform in ('win32',): - return 0 +try: + import resource - try: - return int(os.getenv("PYXMLSEC_TEST_ITERATIONS", "10")) - except ValueError: - return 0 + test_iterations = int(os.environ.get('PYXMLSEC_TEST_ITERATIONS', '10')) +except (ImportError, ValueError): + test_iterations = 0 class TestMemoryLeaks(unittest.TestCase): maxDiff = None - iterations = get_iterations() + iterations = test_iterations data_dir = os.path.join(os.path.dirname(__file__), "data") From e83a4576f0a879acc5594e204bf6a485781d6427 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 11:09:21 +0200 Subject: [PATCH 087/211] [pre-commit.ci] pre-commit autoupdate (#233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 22.6.0 → 22.8.0](https://github.com/psf/black/compare/22.6.0...22.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c89b250..75e4b7ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black types: [] From c4658f37c9df3d9ecd90fa42878b5ae92afa79a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oleg=20H=C3=B6fling?= Date: Thu, 29 Sep 2022 17:47:18 +0200 Subject: [PATCH 088/211] Correct codecov badge URL in readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 03312504..34bdd377 100644 --- a/README.rst +++ b/README.rst @@ -16,8 +16,8 @@ python-xmlsec :target: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml/badge.svg :target: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml -.. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg - :target: https://codecov.io/gh/mehcode/python-xmlsec +.. image:: https://codecov.io/gh/xmlsec/python-xmlsec/branch/master/graph/badge.svg + :target: https://codecov.io/gh/xmlsec/python-xmlsec .. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs :target: https://xmlsec.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status From cc8632f4f3181cdee6f325c47b2dde08117b5f55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:06:32 +0200 Subject: [PATCH 089/211] [pre-commit.ci] pre-commit autoupdate (#238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v0.971 → v0.981](https://github.com/pre-commit/mirrors-mypy/compare/v0.971...v0.981) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75e4b7ab..0545b12f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v0.981 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From 92882809a0c380a8689ab0e7a995803decb2cc59 Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 13:58:25 +0100 Subject: [PATCH 090/211] Added changes to enable 3.11 builds --- .appveyor.yml | 4 ++++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 6 +++--- .github/workflows/opensuse-tumbleweed.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 3 +++ setup.py | 1 + 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..b580525f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..24fa6ddd 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..520e5ba6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,16 +5,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2010_x86_64 - manylinux_2_24_x86_64 - musllinux_1_1_x86_64 exclude: - image: manylinux2010_x86_64 - python-abi: cp310-cp310 + python-abi: cp311-cp311 - image: manylinux2010_i686 - python-abi: cp310-cp310 + python-abi: cp311-cp311 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..273f7f7f 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,7 +6,7 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..d7959208 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/setup.py b/setup.py index 9a3c9277..5c7e0da5 100644 --- a/setup.py +++ b/setup.py @@ -533,6 +533,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From d16f9853e9c85b97428d467b82fcf2fa5c766abf Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:00:32 +0100 Subject: [PATCH 091/211] Added changes to enable 3.11 builds --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b580525f..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ environment: - python: 311 python_version: 3.11.2 - python: 311-x64 - python_version: 3.10.6 + python_version: 3.11.2 install: - ps: | From 519feff7d36389363122472d6a68aa285ed3405d Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:52:40 +0100 Subject: [PATCH 092/211] bumped isort to 5.11.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..6b8fdf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 156394743a0c712e6638fe6e7e300c2f24b4fb12 Mon Sep 17 00:00:00 2001 From: tdivis Date: Fri, 24 Mar 2023 08:03:26 +0100 Subject: [PATCH 093/211] Fix #247 - Fix missing import of `template` in `__init__.pyi` stub. (#248) --- src/xmlsec/__init__.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index be0bd659..56356e55 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -5,6 +5,7 @@ from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element from xmlsec import constants as constants +from xmlsec import template as template from xmlsec import tree as tree from xmlsec.constants import __KeyData as KeyData from xmlsec.constants import __Transform as Transform From a7f95d55cd660d1a212fa76a527063b3b7dbe8bb Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 13:58:25 +0100 Subject: [PATCH 094/211] Added changes to enable 3.11 builds --- .appveyor.yml | 4 ++++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 6 +++--- .github/workflows/opensuse-tumbleweed.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 3 +++ setup.py | 1 + 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..b580525f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..24fa6ddd 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..520e5ba6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,16 +5,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2010_x86_64 - manylinux_2_24_x86_64 - musllinux_1_1_x86_64 exclude: - image: manylinux2010_x86_64 - python-abi: cp310-cp310 + python-abi: cp311-cp311 - image: manylinux2010_i686 - python-abi: cp310-cp310 + python-abi: cp311-cp311 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..273f7f7f 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,7 +6,7 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..d7959208 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/setup.py b/setup.py index 9a3c9277..5c7e0da5 100644 --- a/setup.py +++ b/setup.py @@ -533,6 +533,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From b7683774f747c7aed6a0b30e6045da679bc68760 Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:00:32 +0100 Subject: [PATCH 095/211] Added changes to enable 3.11 builds --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b580525f..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ environment: - python: 311 python_version: 3.11.2 - python: 311-x64 - python_version: 3.10.6 + python_version: 3.11.2 install: - ps: | From bddf28e68a2509a287f9889aaeadc3adab80ccbc Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:52:40 +0100 Subject: [PATCH 096/211] bumped isort to 5.11.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..6b8fdf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 2c58d43eedf72590e3e201252f5fc5ddae36f8c6 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 22 Mar 2023 20:36:45 +0100 Subject: [PATCH 097/211] Fix #244 - Fix failing test with libxmlsec-1.2.36, also make libxmlsec version available from Python. --- src/main.c | 13 ++++ tests/data/sign5-out-xmlsec_1_2_36_to_37.xml | 67 ++++++++++++++++++++ tests/test_ds.py | 6 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/data/sign5-out-xmlsec_1_2_36_to_37.xml diff --git a/src/main.c b/src/main.c index ffcae14f..5773db3b 100644 --- a/src/main.c +++ b/src/main.c @@ -119,6 +119,13 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { Py_RETURN_NONE; } +static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ + "get_libxmlsec_version() -> tuple\n" + "Returns Version tuple of wrapped libxml library."; +static PyObject* PyXmlSec_GetLibXmlSecVersion() { + return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -386,6 +393,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_PyShutdown__doc__ }, + { + "get_libxmlsec_version", + (PyCFunction)PyXmlSec_GetLibXmlSecVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlSecVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml new file mode 100644 index 00000000..f359b138 --- /dev/null +++ b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml @@ -0,0 +1,67 @@ + + + + + Hello, World! + + + + + + + + + + +HjY8ilZAIEM2tBbPn5mYO1ieIX4= + + +SIaj/6KY3C1SmDXU2++Gm31U1xTadFp04WhBgfsJFbxrL+q7GKSKN9kfQ+UpN9+i +D5fWmuavXEHe4Gw6RMaMEkq2URQo7F68+d5J/ajq8/l4n+xE6/reGScVwT6L4dEP +XXVJcAi2ZnQ3O7GTNvNGCPibL9mUcyCWBFZ92Uemtc/vJFCQ7ZyKMdMfACgxOwyN +T/9971oog241/2doudhonc0I/3mgPYWkZdX6yvr62mEjnG+oUZkhWYJ4ewZJ4hM4 +JjbFqZO+OEzDRSbw3DkmuBA/mtlx+3t13SESfEub5hqoMdVmtth/eTb64dsPdl9r +3k1ACVX9f8aHfQQdJOmLFQ== + + + + + + +Test Issuer +1 + +MIIE3zCCBEigAwIBAgIBBTANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tMB4X +DTAzMDMzMTA0MDIyMloXDTEzMDMyODA0MDIyMlowgb8xCzAJBgNVBAYTAlVTMRMw +EQYDVQQIEwpDYWxpZm9ybmlhMT0wOwYDVQQKEzRYTUwgU2VjdXJpdHkgTGlicmFy +eSAoaHR0cDovL3d3dy5hbGVrc2V5LmNvbS94bWxzZWMpMSEwHwYDVQQLExhFeGFt +cGxlcyBSU0EgQ2VydGlmaWNhdGUxFjAUBgNVBAMTDUFsZWtzZXkgU2FuaW4xITAf +BgkqhkiG9w0BCQEWEnhtbHNlY0BhbGVrc2V5LmNvbTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAJe4/rQ/gzV4FokE7CthjL/EXwCBSkXm2c3p4jyXO0Wt +quaNC3dxBwFPfPl94hmq3ZFZ9PHPPbp4RpYRnLZbRjlzVSOq954AXOXpSew7nD+E +mTqQrd9+ZIbGJnLOMQh5fhMVuOW/1lYCjWAhTCcYZPv7VXD2M70vVXDVXn6ZrqTg +qkVHE6gw1aCKncwg7OSOUclUxX8+Zi10v6N6+PPslFc5tKwAdWJhVLTQ4FKG+F53 +7FBDnNK6p4xiWryy/vPMYn4jYGvHUUk3eH4lFTCr+rSuJY8i/KNIf/IKim7g/o3w +Ae3GM8xrof2mgO8GjK/2QDqOQhQgYRIf4/wFsQXVZcMCAwEAAaOCAVcwggFTMAkG +A1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRp +ZmljYXRlMB0GA1UdDgQWBBQkhCzy1FkgYosuXIaQo6owuicanDCB+AYDVR0jBIHw +MIHtgBS0ue+a5pcOaGUemM76VQ2JBttMfKGB0aSBzjCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tggEA +MA0GCSqGSIb3DQEBBAUAA4GBALU/mzIxSv8vhDuomxFcplzwdlLZbvSQrfoNkMGY +1UoS3YJrN+jZLWKSyWE3mIaPpElqXiXQGGkwD5iPQ1iJMbI7BeLvx6ZxX/f+c8Wn +ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL +NJ2D + + + + + diff --git a/tests/test_ds.py b/tests/test_ds.py index 694ad431..38f0b25c 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -182,7 +182,11 @@ def test_sign_case5(self): self.assertEqual("rsakey.pem", ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign5-out.xml"), root) + if (1, 2, 36) <= xmlsec.get_libxmlsec_version() <= (1, 2, 37): + expected_xml_file = 'sign5-out-xmlsec_1_2_36_to_37.xml' + else: + expected_xml_file = 'sign5-out.xml' + self.assertEqual(self.load_xml(expected_xml_file), root) def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() From 1cf6785b3e06c2b8f6cac1266cf8f1934650bc4b Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 21 Dec 2022 14:01:14 +0100 Subject: [PATCH 098/211] Fix #164 - Add support for loading keys from engine (e.g. pkcs11). --- README.rst | 2 +- src/keys.c | 47 +++++++++++++++++++++++++++++++++++++++++ src/xmlsec/__init__.pyi | 2 ++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 34bdd377..e1924652 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Check the `examples `_ se Requirements ************ - ``libxml2 >= 2.9.1`` -- ``libxmlsec1 >= 1.2.18`` +- ``libxmlsec1 >= 1.2.33`` Install ******* diff --git a/src/keys.c b/src/keys.c index 1362b128..1440331c 100644 --- a/src/keys.c +++ b/src/keys.c @@ -185,6 +185,47 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* return NULL; } +static const char PyXmlSec_KeyFromEngine__doc__[] = \ + "from_engine(engine_and_key_id) -> xmlsec.Key\n" + "Loads PKI key from an engine.\n\n" + ":param engine_and_key_id: engine and key id, i.e. 'pkcs11;pkcs11:token=XmlsecToken;object=XmlsecKey;pin-value=password'\n" + ":type engine_and_key_id: :class:`str`, " + ":return: pointer to newly created key\n" + ":rtype: :class:`~xmlsec.Key`"; +static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = {"engine_and_key_id", NULL}; + + const char* engine_and_key_id = NULL; + PyXmlSec_Key* key = NULL; + + PYXMLSEC_DEBUG("load key from engine - start"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s:from_engine", kwlist, &engine_and_key_id)) { + goto ON_FAIL; + } + + if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; + + Py_BEGIN_ALLOW_THREADS; + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), + (void*)engine_and_key_id); + Py_END_ALLOW_THREADS; + + if (key->handle == NULL) { + PyXmlSec_SetLastError("cannot read key"); + goto ON_FAIL; + } + + key->is_own = 1; + + PYXMLSEC_DEBUG("load key from engine - ok"); + return (PyObject*)key; + +ON_FAIL: + PYXMLSEC_DEBUG("load key from engine - fail"); + Py_XDECREF(key); + return NULL; +} + static const char PyXmlSec_KeyGenerate__doc__[] = \ "generate(klass, size, type) -> xmlsec.Key\n" "Generates key of kind ``klass`` with ``size`` and ``type``.\n\n" @@ -494,6 +535,12 @@ static PyMethodDef PyXmlSec_KeyMethods[] = { METH_CLASS|METH_VARARGS|METH_KEYWORDS, PyXmlSec_KeyFromFile__doc__ }, + { + "from_engine", + (PyCFunction)PyXmlSec_KeyFromEngine, + METH_CLASS|METH_VARARGS|METH_KEYWORDS, + PyXmlSec_KeyFromEngine__doc__ + }, { "generate", (PyCFunction)PyXmlSec_KeyGenerate, diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 56356e55..6c326f56 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -49,6 +49,8 @@ class Key: @classmethod def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod + def from_engine(cls: type[Self], engine_and_key_id: AnyStr) -> Self: ... + @classmethod def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... From 0b5939fc65e98cebb669d311b4fb58844bf887e5 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Thu, 13 Apr 2023 14:28:02 +0200 Subject: [PATCH 099/211] Fix #164 - Add tests for pkcs11 (softhsm) key. --- .github/workflows/sdist.yml | 3 +- tests/softhsm_setup.py | 265 ++++++++++++++++++++++++++++++++++++ tests/test_pkcs11.py | 57 ++++++++ 3 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 tests/softhsm_setup.py create mode 100644 tests/test_pkcs11.py diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index d7959208..6ca7a657 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -16,9 +16,8 @@ jobs: run: | python setup.py sdist - name: Install test dependencies - env: - PYXMLSEC_STATIC_DEPS: true run: | + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl pip install --upgrade -r requirements-test.txt pip install black # for stub generation tests pip install dist/xmlsec-$(python setup.py --version).tar.gz diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py new file mode 100644 index 00000000..0a7c37de --- /dev/null +++ b/tests/softhsm_setup.py @@ -0,0 +1,265 @@ +""" +Testing the PKCS#11 shim layer. +Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj +under licence "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import traceback +import unittest +from typing import Dict, List, Optional, Tuple + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") + + +def paths_for_component(component: str, default_paths: List[str]): + env_path = os.environ.get(component) + return [env_path] if env_path else default_paths + + +def find_alts(component_name, alts: List[str]) -> str: + for a in alts: + if os.path.exists(a): + return a + raise unittest.SkipTest("Required component is missing: {}".format(component_name)) + + +def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: + env = {} + if softhsm_conf is not None: + env['SOFTHSM_CONF'] = softhsm_conf + env['SOFTHSM2_CONF'] = softhsm_conf + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + out, err = proc.communicate() + if err is not None and len(err) > 0: + logging.error(err) + if out is not None and len(out) > 0: + logging.debug(out) + rv = proc.wait() + if rv: + with open(softhsm_conf) as f: + conf = f.read() + msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' + msg = msg.format( + cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + ) + raise RuntimeError(msg) + return out, err + + +component_default_paths: Dict[str, List[str]] = { + 'P11_MODULE': [ + '/usr/lib/softhsm/libsofthsm2.so', + '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', + '/usr/lib/softhsm/libsofthsm.so', + '/usr/lib64/softhsm/libsofthsm2.so', + ], + 'P11_ENGINE': [ + '/usr/lib/ssl/engines/libpkcs11.so', + '/usr/lib/engines/engine_pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/libpkcs11.so', + '/usr/lib64/engines-3/pkcs11.so', + '/usr/lib64/engines-3/libpkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/libpkcs11.so', + ], + 'PKCS11_TOOL': [ + '/usr/bin/pkcs11-tool', + ], + 'SOFTHSM': [ + '/usr/bin/softhsm2-util', + '/usr/bin/softhsm', + ], + 'OPENSSL': [ + '/usr/bin/openssl', + ], +} + +component_path: Dict[str, str] = { + component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) + for component_name, default_paths in component_default_paths.items() +} + +softhsm_version = 1 +if component_path['SOFTHSM'].endswith('softhsm2-util'): + softhsm_version = 2 + +openssl_version = subprocess.check_output([component_path['OPENSSL'], + 'version'] + )[8:11].decode() + +p11_test_files: List[str] = [] +softhsm_conf: Optional[str] = None +softhsm_db: Optional[str] = None + + +def _temp_file() -> str: + f = tempfile.NamedTemporaryFile(delete=False) + p11_test_files.append(f.name) + return f.name + + +def _temp_dir() -> str: + d = tempfile.mkdtemp() + p11_test_files.append(d) + return d + + +@unittest.skipIf(component_path['P11_MODULE'] is None, "SoftHSM PKCS11 module not installed") +def setup() -> None: + logging.debug("Creating test pkcs11 token using softhsm") + try: + global softhsm_conf + softhsm_conf = _temp_file() + logging.debug("Generating softhsm.conf") + with open(softhsm_conf, "w") as f: + if softhsm_version == 2: + softhsm_db = _temp_dir() + f.write(""" +# Generated by test +directories.tokendir = %s +objectstore.backend = file +log.level = DEBUG +""" % softhsm_db) + else: + softhsm_db = _temp_file() + f.write(""" +# Generated by test +0:%s +""" % softhsm_db) + + logging.debug("Initializing the token") + out, err = run_cmd([component_path['SOFTHSM'], + '--slot', '0', + '--label', 'test', + '--init-token', + '--pin', 'secret1', + '--so-pin', 'secret2'], + softhsm_conf=softhsm_conf) + + # logging.debug("Generating 1024 bit RSA key in token") + # run_cmd([component_path['PKCS11_TOOL'], + # '--module', component_path['P11_MODULE'], + # '-l', + # '-k', + # '--key-type', 'rsa:1024', + # '--id', 'a1b2', + # '--label', 'test', + # '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + hash_priv_key = _temp_file() + logging.debug("Converting test private key to format for softhsm") + run_cmd([component_path['OPENSSL'], 'pkcs8', + '-topk8', + '-inform', 'PEM', + '-outform', 'PEM', + '-nocrypt', + '-in', os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', hash_priv_key], softhsm_conf=softhsm_conf) + + logging.debug("Importing the test key to softhsm") + run_cmd([component_path['SOFTHSM'], + '--import', hash_priv_key, + '--token', 'test', + '--id', 'a1b2', + '--label', 'test', + '--pin', 'secret1'], + softhsm_conf=softhsm_conf) + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + signer_cert_pem = _temp_file() + openssl_conf = _temp_file() + logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) + with open(openssl_conf, "w") as f: + # Might be needed with some versions of openssl, but in more recent versions dynamic_path breaks it. + # dynamic_path = ( + # "dynamic_path = %s" % component_path['P11_ENGINE'] + # if openssl_version.startswith(b'1.') + # else "" + # ) + f.write("\n".join([ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ])) + + with open(openssl_conf, "r") as f: + logging.debug('-------- START DEBUG openssl_conf --------') + logging.debug(f.readlines()) + logging.debug('-------- END DEBUG openssl_conf --------') + logging.debug('-------- START DEBUG paths --------') + logging.debug(run_cmd(['ls', '-ld', component_path['P11_ENGINE']])) + logging.debug(run_cmd(['ls', '-ld', component_path['P11_MODULE']])) + logging.debug('-------- END DEBUG paths --------') + + signer_cert_der = _temp_file() + + logging.debug("Generating self-signed certificate") + run_cmd([component_path['OPENSSL'], 'req', + '-new', + '-x509', + '-subj', "/CN=Test Signer", + '-engine', 'pkcs11', + '-config', openssl_conf, + '-keyform', 'engine', + '-key', 'label_test', + '-passin', 'pass:secret1', + '-out', signer_cert_pem], softhsm_conf=softhsm_conf) + + run_cmd([component_path['OPENSSL'], 'x509', + '-inform', 'PEM', + '-outform', 'DER', + '-in', signer_cert_pem, + '-out', signer_cert_der], softhsm_conf=softhsm_conf) + + logging.debug("Importing certificate into token") + + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--slot-index', '0', + '--id', 'a1b2', + '--label', 'test', + '-y', 'cert', + '-w', signer_cert_der, + '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + # TODO: Should be teardowned in teardown: + os.environ['SOFTHSM_CONF'] = softhsm_conf + os.environ['SOFTHSM2_CONF'] = softhsm_conf + + except Exception as ex: + print("-" * 64) + traceback.print_exc() + print("-" * 64) + logging.error("PKCS11 tests disabled: unable to initialize test token: %s" % ex) + raise ex + + +def teardown() -> None: + global p11_test_files + for o in p11_test_files: + if os.path.exists(o): + if os.path.isdir(o): + shutil.rmtree(o) + else: + os.unlink(o) + p11_test_files = [] diff --git a/tests/test_pkcs11.py b/tests/test_pkcs11.py new file mode 100644 index 00000000..accd29ae --- /dev/null +++ b/tests/test_pkcs11.py @@ -0,0 +1,57 @@ +import xmlsec +from tests import base +from xmlsec import constants as consts + +KEY_URL = "pkcs11;pkcs11:token=test;object=test;pin-value=secret1" + + +def setUpModule(): + from tests import softhsm_setup + + softhsm_setup.setup() + + +def tearDownModule(): + from tests import softhsm_setup + + softhsm_setup.teardown() + + +class TestKeys(base.TestMemoryLeaks): + def test_del_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + + def test_sign_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaises(TypeError): + ctx.sign('') + + def test_sign_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + ctx.sign(self.load_xml('sign1-in.xml')) + + def test_sign_case1(self): + """Should sign a pre-constructed template file using a key from a pkcs11 engine.""" + root = self.load_xml("sign1-in.xml") + sign = xmlsec.tree.find_node(root, consts.NodeSignature) + self.assertIsNotNone(sign) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + ctx.key.name = 'rsakey.pem' + self.assertEqual("rsakey.pem", ctx.key.name) + + ctx.sign(sign) + self.assertEqual(self.load_xml("sign1-out.xml"), root) From 4f5daea286df89c64fbfc615f422be62b2cdf114 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:55:28 +0000 Subject: [PATCH 100/211] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/softhsm_setup.py | 220 +++++++++++++++++++++++++++-------------- 1 file changed, 147 insertions(+), 73 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 0a7c37de..432d4b1b 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -45,7 +45,11 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: conf = f.read() msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' msg = msg.format( - cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + cmd=" ".join(args), + code=rv, + out=out.strip(), + err=err.strip(), + conf=conf, ) raise RuntimeError(msg) return out, err @@ -90,9 +94,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: if component_path['SOFTHSM'].endswith('softhsm2-util'): softhsm_version = 2 -openssl_version = subprocess.check_output([component_path['OPENSSL'], - 'version'] - )[8:11].decode() +openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() p11_test_files: List[str] = [] softhsm_conf: Optional[str] = None @@ -121,27 +123,41 @@ def setup() -> None: with open(softhsm_conf, "w") as f: if softhsm_version == 2: softhsm_db = _temp_dir() - f.write(""" + f.write( + """ # Generated by test directories.tokendir = %s objectstore.backend = file log.level = DEBUG -""" % softhsm_db) +""" + % softhsm_db + ) else: softhsm_db = _temp_file() - f.write(""" + f.write( + """ # Generated by test 0:%s -""" % softhsm_db) +""" + % softhsm_db + ) logging.debug("Initializing the token") - out, err = run_cmd([component_path['SOFTHSM'], - '--slot', '0', - '--label', 'test', - '--init-token', - '--pin', 'secret1', - '--so-pin', 'secret2'], - softhsm_conf=softhsm_conf) + out, err = run_cmd( + [ + component_path['SOFTHSM'], + '--slot', + '0', + '--label', + 'test', + '--init-token', + '--pin', + 'secret1', + '--so-pin', + 'secret2', + ], + softhsm_conf=softhsm_conf, + ) # logging.debug("Generating 1024 bit RSA key in token") # run_cmd([component_path['PKCS11_TOOL'], @@ -155,26 +171,45 @@ def setup() -> None: hash_priv_key = _temp_file() logging.debug("Converting test private key to format for softhsm") - run_cmd([component_path['OPENSSL'], 'pkcs8', - '-topk8', - '-inform', 'PEM', - '-outform', 'PEM', - '-nocrypt', - '-in', os.path.join(DATA_DIR, 'rsakey.pem'), - '-out', hash_priv_key], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'pkcs8', + '-topk8', + '-inform', + 'PEM', + '-outform', + 'PEM', + '-nocrypt', + '-in', + os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', + hash_priv_key, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing the test key to softhsm") - run_cmd([component_path['SOFTHSM'], - '--import', hash_priv_key, - '--token', 'test', - '--id', 'a1b2', - '--label', 'test', - '--pin', 'secret1'], - softhsm_conf=softhsm_conf) - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['SOFTHSM'], + '--import', + hash_priv_key, + '--token', + 'test', + '--id', + 'a1b2', + '--label', + 'test', + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) + run_cmd( + [component_path['PKCS11_TOOL'], '--module', component_path['P11_MODULE'], '-l', '--pin', 'secret1', '-O'], + softhsm_conf=softhsm_conf, + ) signer_cert_pem = _temp_file() openssl_conf = _temp_file() logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) @@ -185,21 +220,25 @@ def setup() -> None: # if openssl_version.startswith(b'1.') # else "" # ) - f.write("\n".join([ - "openssl_conf = openssl_def", - "[openssl_def]", - "engines = engine_section", - "[engine_section]", - "pkcs11 = pkcs11_section", - "[req]", - "distinguished_name = req_distinguished_name", - "[req_distinguished_name]", - "[pkcs11_section]", - "engine_id = pkcs11", - # dynamic_path, - "MODULE_PATH = %s" % component_path['P11_MODULE'], - "init = 0", - ])) + f.write( + "\n".join( + [ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ] + ) + ) with open(openssl_conf, "r") as f: logging.debug('-------- START DEBUG openssl_conf --------') @@ -213,34 +252,69 @@ def setup() -> None: signer_cert_der = _temp_file() logging.debug("Generating self-signed certificate") - run_cmd([component_path['OPENSSL'], 'req', - '-new', - '-x509', - '-subj', "/CN=Test Signer", - '-engine', 'pkcs11', - '-config', openssl_conf, - '-keyform', 'engine', - '-key', 'label_test', - '-passin', 'pass:secret1', - '-out', signer_cert_pem], softhsm_conf=softhsm_conf) - - run_cmd([component_path['OPENSSL'], 'x509', - '-inform', 'PEM', - '-outform', 'DER', - '-in', signer_cert_pem, - '-out', signer_cert_der], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'req', + '-new', + '-x509', + '-subj', + "/CN=Test Signer", + '-engine', + 'pkcs11', + '-config', + openssl_conf, + '-keyform', + 'engine', + '-key', + 'label_test', + '-passin', + 'pass:secret1', + '-out', + signer_cert_pem, + ], + softhsm_conf=softhsm_conf, + ) + + run_cmd( + [ + component_path['OPENSSL'], + 'x509', + '-inform', + 'PEM', + '-outform', + 'DER', + '-in', + signer_cert_pem, + '-out', + signer_cert_der, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing certificate into token") - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--slot-index', '0', - '--id', 'a1b2', - '--label', 'test', - '-y', 'cert', - '-w', signer_cert_der, - '--pin', 'secret1'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--slot-index', + '0', + '--id', + 'a1b2', + '--label', + 'test', + '-y', + 'cert', + '-w', + signer_cert_der, + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) # TODO: Should be teardowned in teardown: os.environ['SOFTHSM_CONF'] = softhsm_conf From 70753591a9b56542961cb1e3b4cd05c92aea7028 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 12 Jun 2023 14:26:51 +0200 Subject: [PATCH 101/211] Remove SOAP The SOAP support fully removed from xmlsec1 library. --- doc/source/modules/constants.rst | 6 - src/constants.c | 2 - src/xmlsec/constants.pyi | 2 - tests/data/enc-bad-in.xml | 208 ------------------------------- tests/test_enc.py | 19 --- 5 files changed, 237 deletions(-) delete mode 100644 tests/data/enc-bad-in.xml diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 4a63fcd7..8127590a 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -166,12 +166,6 @@ Namespaces .. data:: xmlsec.constants.XPointerNs :annotation: = 'http://www.w3.org/2001/04/xmldsig-more/xptr' -.. data:: xmlsec.constants.Soap11Ns - :annotation: = 'http://schemas.xmlsoap.org/soap/envelope/' - -.. data:: xmlsec.constants.Soap12Ns - :annotation: = 'http://www.w3.org/2002/06/soap-envelope' - .. data:: xmlsec.constants.NsExcC14N :annotation: = 'http://www.w3.org/2001/10/xml-exc-c14n#' diff --git a/src/constants.c b/src/constants.c index 34c81b29..72ae217f 100644 --- a/src/constants.c +++ b/src/constants.c @@ -316,8 +316,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_NS_CONSTANT(XPathNs, "XPATH"); PYXMLSEC_ADD_NS_CONSTANT(XPath2Ns, "XPATH2"); PYXMLSEC_ADD_NS_CONSTANT(XPointerNs, "XPOINTER"); - PYXMLSEC_ADD_NS_CONSTANT(Soap11Ns, "SOAP11"); - PYXMLSEC_ADD_NS_CONSTANT(Soap12Ns, "SOAP12"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14N, "EXC_C14N"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14NWithComments, "EXC_C14N_WITH_COMMENT"); diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3430a027..9fd24e53 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -85,8 +85,6 @@ NodeX509Data: Final[str] Ns: Final[str] NsExcC14N: Final[str] NsExcC14NWithComments: Final[str] -Soap11Ns: Final[str] -Soap12Ns: Final[str] TransformAes128Cbc: Final[__Transform] TransformAes128Gcm: Final[__Transform] TransformAes192Cbc: Final[__Transform] diff --git a/tests/data/enc-bad-in.xml b/tests/data/enc-bad-in.xml deleted file mode 100644 index 460738fc..00000000 --- a/tests/data/enc-bad-in.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - - - - - MyNextCar - CreditApplication - MYNEXTCAR - VW - 409D03 - MyNextCar - - 2018-11-20T09:37:45Z - 7f0842cc-8d47-4955-be31-c61d07ee490b - - VW - - - - - - -
- VCI_MNA_0000070250 - - - Car Chantilly -
- 14839 Stonecroft Center Ct - Chantilly - VA - US - 20151 -
- - - MyNextCar - MNA - - 7039562100 - - CAR - - N -
- - - 2017 - Q7 - CAR - New - 0 - Prestige - - 64300.0 - MSRP - - - 64300.0 - Selling Price - - - - - 113456789 - NationalId - - - John - Q - Public - -
- 999 Washington Ave - Apt #332 - Front Royal - VA - US - 22630 - 01 - 10 - Own -
-
- 21 E 9th Ave - Boulder - CO - US - 80301-7577 - 07 - 11 - Own -
- - 3032852402 - 3032852405 - 7203554444 - JohnQPublic@anydomain.org - - - 1967-07-31 - - 0 - - UPS -
- 1775 Wiehle Ave. - Reston - VA - US - 20190 -
- 9500.0 - Driver - 01 - 05 - Current - - - FedEx - 4000.00 - Driver - 04 - 09 - Previous - - 1252.52 - - 1500.00 - - - 1 - Consents to Credit Check - - - - - 123435325 - NationalId - - - Lisa - C - Public - -
- 999 Lewis Street - Front Royal - VA - US - 22630 - 5 - 0 - Own -
- - 5401110000 - 5401110073 - public@test.com - - - 1963-04-20 - - - Christendom College -
- 999 Christendom Dr - Front Royal - VA - US - 22630 -
- 6200.00 - Professor - 5 - 0 - Current - - 1252.52 - - 1 - Consents to Credit Check - - - - R - 0.00 - 66 - 5000.00 - INDIVCOAPP - 2000.00 - MyNextCar - - 1978 - Bonneville - Pontiac - Coupe - - - - - - - - diff --git a/tests/test_enc.py b/tests/test_enc.py index 63e00169..1788b4d6 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -233,22 +233,3 @@ def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): ctx.decrypt('') - - def check_no_segfault(self): - namespaces = {'soap': 'http://schemas.xmlsoap.org/soap/envelope/'} - - manager = xmlsec.KeysManager() - key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) - manager.add_key(key) - template = self.load_xml('enc-bad-in.xml') - enc_data = xmlsec.template.encrypted_data_create( - template, xmlsec.Transform.AES128, type=xmlsec.EncryptionType.CONTENT, ns='xenc' - ) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') - enc_key = xmlsec.template.add_encrypted_key(key_info, xmlsec.Transform.RSA_PKCS1) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_key) - data = template.find('soap:Body', namespaces=namespaces) - enc_ctx = xmlsec.EncryptionContext(manager) - enc_ctx.key = xmlsec.Key.generate(xmlsec.KeyData.AES, 192, xmlsec.KeyDataType.SESSION) - self.assertRaises(Exception, enc_ctx.encrypt_xml(enc_data, data)) From 2c26131e2d965346b538d86349fe121027afd9f4 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 12 Jun 2023 15:12:57 +0200 Subject: [PATCH 102/211] Upgrade isort because of poetry dependency issue Fix https://github.com/PyCQA/isort/issues/2077 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..bf877f39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From b865569c84ff00f116b321527a2bd381321c9402 Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Thu, 24 Aug 2023 00:29:17 +0300 Subject: [PATCH 103/211] setup: better deal with compiler flags - honor environment variable PYXMLSEC_OPTIMIZE_SIZE to switch between speed and size optimization - enabled by default for backward compatibility - better deal with compiler flags for different platforms Signed-off-by: Konstantin Demin --- setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9a3c9277..c0fd0f7d 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ext = self.ext_map['xmlsec'] self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) + self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) if self.static or sys.platform == 'win32': self.info('starting static build on {}'.format(sys.platform)) @@ -153,11 +154,18 @@ def run(self): ) if self.debug: - ext.extra_compile_args.append('-Wall') - ext.extra_compile_args.append('-O0') ext.define_macros.append(('PYXMLSEC_ENABLE_DEBUG', '1')) + if sys.platform == 'win32': + ext.extra_compile_args.append('/Od') + else: + ext.extra_compile_args.append('-Wall') + ext.extra_compile_args.append('-O0') else: - ext.extra_compile_args.append('-Os') + if self.size_opt: + if sys.platform == 'win32': + ext.extra_compile_args.append('/Os') + else: + ext.extra_compile_args.append('-Os') super(build_ext, self).run() From 5f57f2eac756df213e83a2fdb452057909e2dd21 Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 17:49:15 -0700 Subject: [PATCH 104/211] Add compatibility for xmlsec 1.3 --- doc/source/modules/constants.rst | 6 ------ src/constants.c | 2 -- src/enc.c | 7 +++++++ src/xmlsec/constants.pyi | 2 -- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 4a63fcd7..8127590a 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -166,12 +166,6 @@ Namespaces .. data:: xmlsec.constants.XPointerNs :annotation: = 'http://www.w3.org/2001/04/xmldsig-more/xptr' -.. data:: xmlsec.constants.Soap11Ns - :annotation: = 'http://schemas.xmlsoap.org/soap/envelope/' - -.. data:: xmlsec.constants.Soap12Ns - :annotation: = 'http://www.w3.org/2002/06/soap-envelope' - .. data:: xmlsec.constants.NsExcC14N :annotation: = 'http://www.w3.org/2001/10/xml-exc-c14n#' diff --git a/src/constants.c b/src/constants.c index 34c81b29..72ae217f 100644 --- a/src/constants.c +++ b/src/constants.c @@ -316,8 +316,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_NS_CONSTANT(XPathNs, "XPATH"); PYXMLSEC_ADD_NS_CONSTANT(XPath2Ns, "XPATH2"); PYXMLSEC_ADD_NS_CONSTANT(XPointerNs, "XPOINTER"); - PYXMLSEC_ADD_NS_CONSTANT(Soap11Ns, "SOAP11"); - PYXMLSEC_ADD_NS_CONSTANT(Soap12Ns, "SOAP12"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14N, "EXC_C14N"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14NWithComments, "EXC_C14N_WITH_COMMENT"); diff --git a/src/enc.c b/src/enc.c index aaf35ae5..475cd2d4 100644 --- a/src/enc.c +++ b/src/enc.c @@ -50,6 +50,13 @@ static int PyXmlSec_EncryptionContext__init__(PyObject* self, PyObject* args, Py } ctx->manager = manager; PYXMLSEC_DEBUGF("%p: init enc context - ok, manager - %p", self, manager); + + // xmlsec 1.3 changed the key search to strict mode, causing various examples + // in the docs to fail. For backwards compatibility, this changes it back to + // lax mode for now. + ctx->handle->keyInfoReadCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + ctx->handle->keyInfoWriteCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + return 0; ON_FAIL: PYXMLSEC_DEBUGF("%p: init enc context - failed", self); diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3430a027..9fd24e53 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -85,8 +85,6 @@ NodeX509Data: Final[str] Ns: Final[str] NsExcC14N: Final[str] NsExcC14NWithComments: Final[str] -Soap11Ns: Final[str] -Soap12Ns: Final[str] TransformAes128Cbc: Final[__Transform] TransformAes128Gcm: Final[__Transform] TransformAes192Cbc: Final[__Transform] From 0e7c5e718e8d221331b0a5f851094a5b4181eeae Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 18:07:07 -0700 Subject: [PATCH 105/211] fix 1.2 compatibility --- src/enc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/enc.c b/src/enc.c index 475cd2d4..5453ef99 100644 --- a/src/enc.c +++ b/src/enc.c @@ -17,6 +17,11 @@ #include #include +// Backwards compatibility with xmlsec 1.2 +#ifndef XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH +#define XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH 0x00008000 +#endif + typedef struct { PyObject_HEAD xmlSecEncCtxPtr handle; From 40c14c45a352f2812e796a071ef7d12f5a70be58 Mon Sep 17 00:00:00 2001 From: David Adamec Date: Fri, 15 Sep 2023 19:40:04 -0700 Subject: [PATCH 106/211] update isort to fix poetry error in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..bf877f39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From f2617835b9c0dc8740caa89e7aa765e3d5afe445 Mon Sep 17 00:00:00 2001 From: Chris Novakovic Date: Thu, 12 Oct 2023 18:09:38 +0100 Subject: [PATCH 107/211] Don't define `MODULE_NAME` as a string literal The `MODULE_NAME` macro is used in contexts where a string literal is not valid, but the fallback value set in `src/common.h` defines it as such; this differs from how it is defined in `setup.py`. Define `MODULE_NAME` in `src/common.h` as it is defined in `setup.py`. Fixes #267. --- src/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common.h b/src/common.h index 243ed651..a6176551 100644 --- a/src/common.h +++ b/src/common.h @@ -13,7 +13,7 @@ #include "debug.h" #ifndef MODULE_NAME -#define MODULE_NAME "xmlsec" +#define MODULE_NAME xmlsec #endif #define JOIN(X,Y) DO_JOIN1(X,Y) From c97f2b8ddc7b30de73411bdfbeb9903899ee1495 Mon Sep 17 00:00:00 2001 From: Chris Novakovic Date: Fri, 13 Oct 2023 10:53:48 +0100 Subject: [PATCH 108/211] Make DES/3DES/KW-3DES support conditional on DES availability in XMLSec Some TLS libraries (e.g. OpenSSL) can be built without support for DES and DES-derived algorithms. Rather than assuming that xmlsec always supports them, guard the declaration of the DES, 3DES and KW-3DES constants based on the value of `XMLSEC_NO_DES`. Fixes #269. --- src/constants.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/constants.c b/src/constants.c index 34c81b29..d797086d 100644 --- a/src/constants.c +++ b/src/constants.c @@ -441,7 +441,9 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRetrievalMethod, "RETRIEVALMETHOD") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEncryptedKey, "ENCRYPTEDKEY") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataAes, "AES") +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDes, "DES") +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif @@ -489,8 +491,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes192, "KW_AES192"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes256, "KW_AES256"); +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDes3Cbc, "DES3"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWDes3, "KW_DES3"); +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDsaSha1, "DSA_SHA1"); #endif From 31bea7c0d2694150818f9d901682aa1fe84449ac Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Sat, 13 Jan 2024 19:40:05 -0500 Subject: [PATCH 109/211] Use KeyDataEc rather than deprecated KeyDataEcdsa --- doc/source/modules/constants.rst | 2 +- src/constants.c | 2 +- src/xmlsec/constants.pyi | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 8127590a..8fdac118 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -47,7 +47,7 @@ KeyData The DSA key klass. -.. data:: xmlsec.constants.KeyDataEcdsa +.. data:: xmlsec.constants.KeyDataEc The ECDSA key klass. diff --git a/src/constants.c b/src/constants.c index e3b64527..f2c5a636 100644 --- a/src/constants.c +++ b/src/constants.c @@ -447,7 +447,7 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #endif #if XMLSEC_VERSION_HEX > 0x10212 // from version 1.2.19 - PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") + PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEc, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRsa, "RSA") diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 9fd24e53..80afdd22 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -29,7 +29,7 @@ EncNs: Final[str] KeyDataAes: Final[__KeyData] KeyDataDes: Final[__KeyData] KeyDataDsa: Final[__KeyData] -KeyDataEcdsa: Final[__KeyData] +KeyDataEc: Final[__KeyData] KeyDataEncryptedKey: Final[__KeyData] KeyDataFormatBinary: Final[int] KeyDataFormatCertDer: Final[int] From 7891e715a7c343a1fd52c86a5085a1d1cc62e711 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Sat, 13 Jan 2024 19:40:28 -0500 Subject: [PATCH 110/211] Use xmlSecCryptoAppKeyLoadEx instead of deprecated xmlSecCryptoAppKeyLoad --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index 1362b128..47678a53 100644 --- a/src/keys.c +++ b/src/keys.c @@ -163,7 +163,7 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* if (is_content) { key->handle = xmlSecCryptoAppKeyLoadMemory((const xmlSecByte*)data, (xmlSecSize)data_size, format, password, NULL, NULL); } else { - key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); } Py_END_ALLOW_THREADS; From c9c660b8f336a8501ab655b9f59a993378b5f38b Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 12 Mar 2024 19:51:08 -0400 Subject: [PATCH 111/211] Use xmlSecCryptoAppKeyLoadEx instead of xmlSecCryptoAppKeyLoad for pkcs11 support --- src/keys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys.c b/src/keys.c index a8ffde25..8b84c343 100644 --- a/src/keys.c +++ b/src/keys.c @@ -206,7 +206,7 @@ static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; Py_BEGIN_ALLOW_THREADS; - key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), + key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); Py_END_ALLOW_THREADS; From 595aeaa2651ba0a1f80565ceac7550fd1abd38fe Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Wed, 13 Mar 2024 16:46:55 +0100 Subject: [PATCH 112/211] Make the library backward compatible Starting from version xmlsec1-1.3.3, some of the constant and keys that were deprecated, removed from the library entirely. This change would add version awareness and backward compatible. --- doc/source/modules/constants.rst | 6 +++++- src/constants.c | 7 +++++-- src/keys.c | 15 ++++++++++++--- src/xmlsec/constants.pyi | 1 + 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 8fdac118..3df6b50f 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -47,9 +47,13 @@ KeyData The DSA key klass. +.. data:: xmlsec.constants.KeyDataEcdsa + + (Deprecated. The EC key klass) The ECDSA key klass. + .. data:: xmlsec.constants.KeyDataEc - The ECDSA key klass. + The EC key klass. .. data:: xmlsec.constants.KeyDataHmac diff --git a/src/constants.c b/src/constants.c index f2c5a636..bd1fa5e0 100644 --- a/src/constants.c +++ b/src/constants.c @@ -445,8 +445,11 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif -#if XMLSEC_VERSION_HEX > 0x10212 - // from version 1.2.19 +#if XMLSEC_VERSION_HEX > 0x10212 && XMLSEC_VERSION_HEX < 0x10303 + // from version 1.2.19 to version 1.3.2 (inclusive) + PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") +#elif XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEc, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") diff --git a/src/keys.c b/src/keys.c index 8b84c343..5ff04aae 100644 --- a/src/keys.c +++ b/src/keys.c @@ -163,7 +163,12 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* if (is_content) { key->handle = xmlSecCryptoAppKeyLoadMemory((const xmlSecByte*)data, (xmlSecSize)data_size, format, password, NULL, NULL); } else { - key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); + #else + key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + #endif } Py_END_ALLOW_THREADS; @@ -206,8 +211,12 @@ static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; Py_BEGIN_ALLOW_THREADS; - key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), - (void*)engine_and_key_id); + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #else + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #endif Py_END_ALLOW_THREADS; if (key->handle == NULL) { diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 80afdd22..3c3ea94f 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -30,6 +30,7 @@ KeyDataAes: Final[__KeyData] KeyDataDes: Final[__KeyData] KeyDataDsa: Final[__KeyData] KeyDataEc: Final[__KeyData] +KeyDataEcdsa: Final[__KeyData] KeyDataEncryptedKey: Final[__KeyData] KeyDataFormatBinary: Final[int] KeyDataFormatCertDer: Final[int] From 802dff28340d9031d694ad77825a7333dc98ef93 Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Wed, 13 Mar 2024 16:17:34 -0400 Subject: [PATCH 113/211] Update correct URL: No func change --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e1924652..60fb3ea1 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Building from source .. code-block:: bash - git clone https://github.com/mehcode/python-xmlsec.git + git clone https://github.com/xmlsec/python-xmlsec.git #. Change into the ``python-xmlsec`` root directory. From 0bad2e462da4ea912c7a258334d0bcb3e0c5aa86 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 17 Mar 2024 13:25:15 -0300 Subject: [PATCH 114/211] Add ability to query libxml version number --- src/main.c | 15 ++++++++++++++- src/platform.h | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main.c b/src/main.c index 5773db3b..0ee5e2cd 100644 --- a/src/main.c +++ b/src/main.c @@ -121,11 +121,18 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ "get_libxmlsec_version() -> tuple\n" - "Returns Version tuple of wrapped libxml library."; + "Returns Version tuple of wrapped libxmlsec library."; static PyObject* PyXmlSec_GetLibXmlSecVersion() { return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); } +static char PyXmlSec_GetLibXmlVersion__doc__[] = \ + "get_libxml_version() -> tuple\n" + "Returns Version tuple of wrapped libxml library."; +static PyObject* PyXmlSec_GetLibXmlVersion() { + return Py_BuildValue("(iii)", XMLSEC_LIBXML_VERSION_MAJOR, XMLSEC_LIBXML_VERSION_MINOR, XMLSEC_LIBXML_VERSION_PATCH); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -399,6 +406,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_GetLibXmlSecVersion__doc__ }, + { + "get_libxml_version", + (PyCFunction)PyXmlSec_GetLibXmlVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/src/platform.h b/src/platform.h index 35163e88..1fc82b7b 100644 --- a/src/platform.h +++ b/src/platform.h @@ -12,6 +12,7 @@ #define PY_SSIZE_T_CLEAN 1 +#include #include #include @@ -19,6 +20,12 @@ #include #endif /* MS_WIN32 */ +#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) + +#define XMLSEC_LIBXML_VERSION_MAJOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100 * 100) +#define XMLSEC_LIBXML_VERSION_MINOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100) +#define XMLSEC_LIBXML_VERSION_PATCH XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 1) + #define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 16) | (XMLSEC_VERSION_MINOR << 8) | (XMLSEC_VERSION_SUBMINOR)) // XKMS support was removed in version 1.2.21 From 01b18c9a09ee54a3d78c8da039777de8512d7380 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 20:35:34 -0300 Subject: [PATCH 115/211] Set enable-md5 when doing static build for libxmlsec1 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 489844c8..277a8892 100644 --- a/setup.py +++ b/setup.py @@ -437,6 +437,7 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-shared', '--disable-gost', + '--enable-md5', '--disable-crypto-dl', '--enable-static=yes', '--enable-shared=no', From 2a717dd3987ba9538617874aba36e4f8326f514e Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:25:01 -0300 Subject: [PATCH 116/211] Fix sdist workflow --- .github/workflows/sdist.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 6ca7a657..e7c0f39d 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -18,8 +18,7 @@ jobs: - name: Install test dependencies run: | sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl - pip install --upgrade -r requirements-test.txt - pip install black # for stub generation tests + pip install --upgrade -r requirements-test.txt --no-binary lxml pip install dist/xmlsec-$(python setup.py --version).tar.gz - name: Run tests run: | From e8ff43b50faf940637d0627b7de2f4566f0dfe3d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 22:05:04 -0300 Subject: [PATCH 117/211] Fix macos workflow --- .github/workflows/macosx.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 24fa6ddd..2c974831 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -19,8 +19,9 @@ jobs: - name: Build macosx_x86_64 wheel env: CC: clang - CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -I/usr/local/opt/libxml2/include" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping -L/usr/local/opt/libxml2/lib" + PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" run: | python -m build rm -rf build/ @@ -31,9 +32,13 @@ jobs: echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies run: | - pip install coverage --upgrade -r requirements-test.txt + pip install coverage --upgrade -r requirements-test.txt --no-binary lxml pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV + env: + CFLAGS: "-I/usr/local/opt/libxml2/include" + LDFLAGS: "-L/usr/local/opt/libxml2/lib" + PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" - name: Run tests run: | coverage run -m pytest -v --color=yes From 5c2cbd59b936bc5fe8563c4cfa9a1b350b5fae3f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:08:36 -0300 Subject: [PATCH 118/211] Add arch flag --- .github/workflows/macosx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 2c974831..39fc37de 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -36,6 +36,7 @@ jobs: pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV env: + CC: clang CFLAGS: "-I/usr/local/opt/libxml2/include" LDFLAGS: "-L/usr/local/opt/libxml2/lib" PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" @@ -45,5 +46,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov -arch x86_64 show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From ad0b27d3ac3f1d89b4641dfdc1482d54d9dd20a1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:13:53 -0300 Subject: [PATCH 119/211] No need to install lxml from source, brew libxml2 version matches --- .github/workflows/macosx.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 39fc37de..129c4e89 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -32,14 +32,9 @@ jobs: echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies run: | - pip install coverage --upgrade -r requirements-test.txt --no-binary lxml + pip install coverage --upgrade -r requirements-test.txt pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV - env: - CC: clang - CFLAGS: "-I/usr/local/opt/libxml2/include" - LDFLAGS: "-L/usr/local/opt/libxml2/lib" - PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" - name: Run tests run: | coverage run -m pytest -v --color=yes From 4bd490143932efaaf41f25ea3b3f3866cbe50554 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 09:16:54 -0300 Subject: [PATCH 120/211] fix arch flag --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 129c4e89..2bc44c1e 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -41,5 +41,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov -arch x86_64 show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=x86_64 --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From 869beeb281cc1366a63e29f7145c62082fbacbef Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:28:25 -0300 Subject: [PATCH 121/211] Fix manylinux build --- .github/workflows/manylinux.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 520e5ba6..6d2dbb1a 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -7,14 +7,7 @@ jobs: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - - manylinux2010_x86_64 - - manylinux_2_24_x86_64 - - musllinux_1_1_x86_64 - exclude: - - image: manylinux2010_x86_64 - python-abi: cp311-cp311 - - image: manylinux2010_i686 - python-abi: cp311-cp311 + - manylinux2014_x86_64 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 From 5794266cff235c0788842c49815408ed35f04e06 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 11:19:59 -0300 Subject: [PATCH 122/211] Try just setting pkg_config_path. --- .github/workflows/macosx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 2bc44c1e..e2a04057 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -19,8 +19,8 @@ jobs: - name: Build macosx_x86_64 wheel env: CC: clang - CFLAGS: "-fprofile-instr-generate -fcoverage-mapping -I/usr/local/opt/libxml2/include" - LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping -L/usr/local/opt/libxml2/lib" + CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" run: | python -m build From ee9fbd03ac4c466447f0f67e3032866657dcb19d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 21 Mar 2024 20:52:49 -0300 Subject: [PATCH 123/211] Version check when setting up lxml - Expose both the compiled version and the linked version of libxml2 - Do a check that the versions match when initializing the module. Otherwise raise an exception. This seems like a better result then a difficult to diagnose segfault. --- src/lxml.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lxml.h | 9 +++++ src/main.c | 30 ++++++++++++++-- src/platform.h | 7 ---- 4 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index aa1abae0..bb7e6a96 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -9,6 +9,7 @@ #include "common.h" #include "lxml.h" +#include "exception.h" #include #include @@ -17,8 +18,102 @@ #include #include +#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) + +#define XMLSEC_EXTRACT_MAJOR(x) XMLSEC_EXTRACT_VERSION(x, 100 * 100) +#define XMLSEC_EXTRACT_MINOR(x) XMLSEC_EXTRACT_VERSION(x, 100) +#define XMLSEC_EXTRACT_PATCH(x) XMLSEC_EXTRACT_VERSION(x, 1) + +static long PyXmlSec_GetLibXmlVersionLong() { + return PyOS_strtol(xmlParserVersion, NULL, 10); +} +long PyXmlSec_GetLibXmlVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionMinor() { + return XMLSEC_EXTRACT_MINOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionPatch() { + return XMLSEC_EXTRACT_PATCH(PyXmlSec_GetLibXmlVersionLong()); +} + +long PyXmlSec_GetLibXmlCompiledVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionMinor() { + return XMLSEC_EXTRACT_MINOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionPatch() { + return XMLSEC_EXTRACT_PATCH(LIBXML_VERSION); +} + +static int PyXmlSec_CheckLibXmlLibraryVersion(void) { + // Make sure that the version of libxml2 that we were compiled against is the same as the one + // that is loaded. If there is a version mismatch, we could run into segfaults. + + if (PyXmlSec_GetLibXmlVersionMajor() != PyXmlSec_GetLibXmlCompiledVersionMajor() || + PyXmlSec_GetLibXmlVersionMinor() != PyXmlSec_GetLibXmlCompiledVersionMinor()) { + return -1; + } + + return 0; +} + +static int PyXmlSec_CheckLxmlLibraryVersion(void) { + // Make sure that the version of libxml2 lxml is using is the same as the one we are using. Because + // we pass trees between the two libraries, we need to make sure that they are using the same version + // of libxml2, or we could run into difficult to debug segfaults. + // See: https://github.com/xmlsec/python-xmlsec/issues/283 + + PyObject* lxml = NULL; + PyObject* version = NULL; + + // Default to failure + int result = -1; + + lxml = PyImport_ImportModule("lxml.etree"); + if (lxml == NULL) { + goto FINALIZE; + } + version = PyObject_GetAttrString(lxml, "LIBXML_VERSION"); + if (version == NULL) { + goto FINALIZE; + } + if (!PyTuple_Check(version) || PyTuple_Size(version) != 3) { + goto FINALIZE; + } + + PyObject* major = PyTuple_GetItem(version, 0); + PyObject* minor = PyTuple_GetItem(version, 1); + + if (!PyLong_Check(major) || !PyLong_Check(minor)) { + goto FINALIZE; + } + + if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { + goto FINALIZE; + } + + result = 0; + +FINALIZE: + // Cleanup our references, and return the result + Py_XDECREF(lxml); + Py_XDECREF(version); + return result; +} int PyXmlSec_InitLxmlModule(void) { + if (PyXmlSec_CheckLibXmlLibraryVersion() < 0) { + PyXmlSec_SetLastError("xmlsec libxml2 library compiled version vs runtime version mismatch"); + return -1; + } + + if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { + PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); + return -1; + } + return import_lxml__etree(); } diff --git a/src/lxml.h b/src/lxml.h index 6824076b..72050efe 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -29,4 +29,13 @@ PyXmlSec_LxmlElementPtr PyXmlSec_elementFactory(PyXmlSec_LxmlDocumentPtr doc, xm // converts o to PyObject, None object is not allowed, does not increment ref_counts int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); +// get version numbers for libxml2 both compiled and loaded +long PyXmlSec_GetLibXmlVersionMajor(); +long PyXmlSec_GetLibXmlVersionMinor(); +long PyXmlSec_GetLibXmlVersionPatch(); + +long PyXmlSec_GetLibXmlCompiledVersionMajor(); +long PyXmlSec_GetLibXmlCompiledVersionMinor(); +long PyXmlSec_GetLibXmlCompiledVersionPatch(); + #endif // __PYXMLSEC_LXML_H__ diff --git a/src/main.c b/src/main.c index 0ee5e2cd..61eac139 100644 --- a/src/main.c +++ b/src/main.c @@ -10,6 +10,7 @@ #include "common.h" #include "platform.h" #include "exception.h" +#include "lxml.h" #include #include @@ -127,10 +128,27 @@ static PyObject* PyXmlSec_GetLibXmlSecVersion() { } static char PyXmlSec_GetLibXmlVersion__doc__[] = \ - "get_libxml_version() -> tuple\n" - "Returns Version tuple of wrapped libxml library."; + "get_libxml_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec is using."; static PyObject* PyXmlSec_GetLibXmlVersion() { - return Py_BuildValue("(iii)", XMLSEC_LIBXML_VERSION_MAJOR, XMLSEC_LIBXML_VERSION_MINOR, XMLSEC_LIBXML_VERSION_PATCH); + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlVersionMajor(), + PyXmlSec_GetLibXmlVersionMinor(), + PyXmlSec_GetLibXmlVersionPatch() + ); +} + +static char PyXmlSec_GetLibXmlCompiledVersion__doc__[] = \ + "get_libxml_compiled_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec was compiled with."; +static PyObject* PyXmlSec_GetLibXmlCompiledVersion() { + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlCompiledVersionMajor(), + PyXmlSec_GetLibXmlCompiledVersionMinor(), + PyXmlSec_GetLibXmlCompiledVersionPatch() + ); } static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ @@ -412,6 +430,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_GetLibXmlVersion__doc__ }, + { + "get_libxml_compiled_version", + (PyCFunction)PyXmlSec_GetLibXmlCompiledVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlCompiledVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/src/platform.h b/src/platform.h index 1fc82b7b..35163e88 100644 --- a/src/platform.h +++ b/src/platform.h @@ -12,7 +12,6 @@ #define PY_SSIZE_T_CLEAN 1 -#include #include #include @@ -20,12 +19,6 @@ #include #endif /* MS_WIN32 */ -#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) - -#define XMLSEC_LIBXML_VERSION_MAJOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100 * 100) -#define XMLSEC_LIBXML_VERSION_MINOR XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 100) -#define XMLSEC_LIBXML_VERSION_PATCH XMLSEC_EXTRACT_VERSION(LIBXML_VERSION, 1) - #define XMLSEC_VERSION_HEX ((XMLSEC_VERSION_MAJOR << 16) | (XMLSEC_VERSION_MINOR << 8) | (XMLSEC_VERSION_SUBMINOR)) // XKMS support was removed in version 1.2.21 From d8d6c6266a1a9c3deed4a8b809325c6549c84530 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:45:23 -0300 Subject: [PATCH 124/211] Update .github/workflows/macosx.yml Co-authored-by: Stu Tomlinson --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index e2a04057..f1bfb7d0 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -21,7 +21,7 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - PKG_CONFIG_PATH: "/usr/local/opt/libxml2/lib/pkgconfig" + PKG_CONFIG_PATH: "$(brew --prefix)/opt/libxml2/lib/pkgconfig" run: | python -m build rm -rf build/ From d84f015f6aa4a6b058c4b0ef5ca9044f39b512f8 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:45:32 -0300 Subject: [PATCH 125/211] Update .github/workflows/macosx.yml Co-authored-by: Stu Tomlinson --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index f1bfb7d0..9b5149c1 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -41,5 +41,5 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=x86_64 --instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt From 47cb969eeb99d9435fb9e3eb063f5dd2490f220b Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 24 Mar 2024 19:56:06 -0300 Subject: [PATCH 126/211] Move env var definition --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 9b5149c1..c477157c 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -21,8 +21,8 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" - PKG_CONFIG_PATH: "$(brew --prefix)/opt/libxml2/lib/pkgconfig" run: | + export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build rm -rf build/ - name: Set environment variables From 497971cfe6d0f79cbc54f59f6c4b93f740c27c53 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 25 Mar 2024 07:17:16 -0300 Subject: [PATCH 127/211] Remove PyXmlSec_CheckLibXmlLibraryVersion check --- src/lxml.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index bb7e6a96..6286cacb 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -47,18 +47,6 @@ long PyXmlSec_GetLibXmlCompiledVersionPatch() { return XMLSEC_EXTRACT_PATCH(LIBXML_VERSION); } -static int PyXmlSec_CheckLibXmlLibraryVersion(void) { - // Make sure that the version of libxml2 that we were compiled against is the same as the one - // that is loaded. If there is a version mismatch, we could run into segfaults. - - if (PyXmlSec_GetLibXmlVersionMajor() != PyXmlSec_GetLibXmlCompiledVersionMajor() || - PyXmlSec_GetLibXmlVersionMinor() != PyXmlSec_GetLibXmlCompiledVersionMinor()) { - return -1; - } - - return 0; -} - static int PyXmlSec_CheckLxmlLibraryVersion(void) { // Make sure that the version of libxml2 lxml is using is the same as the one we are using. Because // we pass trees between the two libraries, we need to make sure that they are using the same version @@ -104,11 +92,6 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { } int PyXmlSec_InitLxmlModule(void) { - if (PyXmlSec_CheckLibXmlLibraryVersion() < 0) { - PyXmlSec_SetLastError("xmlsec libxml2 library compiled version vs runtime version mismatch"); - return -1; - } - if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); return -1; From c7e3208822245f5a3cc1dd04cdeb11125be3945f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 25 Mar 2024 11:08:36 -0300 Subject: [PATCH 128/211] Add type hints --- src/xmlsec/__init__.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 6c326f56..9cfc8cc6 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -13,6 +13,8 @@ from xmlsec.constants import __Transform as Transform _E = TypeVar('_E', bound=_Element) def enable_debug_trace(enabled: bool = ...) -> None: ... +def get_libxml_version() -> tuple[int, int, int]: ... +def get_libxml_compiled_version() -> tuple[int, int, int]: ... def init() -> None: ... def shutdown() -> None: ... def cleanup_callbacks() -> None: ... From e7cbb7c8c40698a7b75c57cd59c47d356c2485e7 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 27 Mar 2024 09:36:17 -0300 Subject: [PATCH 129/211] Update checks based on some code review feedback. --- src/lxml.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 6286cacb..7be62dad 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -67,18 +67,31 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { if (version == NULL) { goto FINALIZE; } - if (!PyTuple_Check(version) || PyTuple_Size(version) != 3) { + if (!PyTuple_Check(version) || PyTuple_Size(version) < 2) { goto FINALIZE; } PyObject* major = PyTuple_GetItem(version, 0); PyObject* minor = PyTuple_GetItem(version, 1); + if (PyErr_Occurred()) { + goto FINALIZE; + } + if (!PyLong_Check(major) || !PyLong_Check(minor)) { goto FINALIZE; } - if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { + long lxml_major = PyLong_AsLong(major); + long lxml_minor = PyLong_AsLong(minor); + long xmlsec_major = PyXmlSec_GetLibXmlVersionMajor(); + long xmlsec_minor = PyXmlSec_GetLibXmlVersionMinor(); + + if (PyErr_Occurred()) { + goto FINALIZE; + } + + if (lxml_major != xmlsec_major || lxml_minor != xmlsec_minor) { goto FINALIZE; } @@ -88,6 +101,10 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { // Cleanup our references, and return the result Py_XDECREF(lxml); Py_XDECREF(version); + + // Clear any errors that may have occurred + PyErr_Clear(); + return result; } From 802904edd0edb09cbfab854a527622c4d3019ff1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:36:20 -0300 Subject: [PATCH 130/211] Make sure exception is handled when getting tuple item Co-authored-by: scoder --- src/lxml.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 7be62dad..f5b74b6a 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -72,9 +72,11 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { } PyObject* major = PyTuple_GetItem(version, 0); + if (major == NULL) { + goto FINALIZE; + } PyObject* minor = PyTuple_GetItem(version, 1); - - if (PyErr_Occurred()) { + if (minor == NULL) { goto FINALIZE; } From 8ca3bdca7b825a92bbdcbffec643429266891b34 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:37:24 -0300 Subject: [PATCH 131/211] Revert changes to version comparison Co-authored-by: scoder --- src/lxml.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index f5b74b6a..02211a3a 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -84,16 +84,7 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { goto FINALIZE; } - long lxml_major = PyLong_AsLong(major); - long lxml_minor = PyLong_AsLong(minor); - long xmlsec_major = PyXmlSec_GetLibXmlVersionMajor(); - long xmlsec_minor = PyXmlSec_GetLibXmlVersionMinor(); - - if (PyErr_Occurred()) { - goto FINALIZE; - } - - if (lxml_major != xmlsec_major || lxml_minor != xmlsec_minor) { + if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { goto FINALIZE; } From b184cfe8f6b1a9592ff3e58aa2a93067d49bd5d4 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 28 Mar 2024 09:37:59 -0300 Subject: [PATCH 132/211] Clear any potential exceptions before Py_XDECREF Co-authored-by: scoder --- src/lxml.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lxml.c b/src/lxml.c index 02211a3a..c98e933b 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -91,13 +91,13 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { result = 0; FINALIZE: + // Clear any errors that may have occurred + PyErr_Clear(); + // Cleanup our references, and return the result Py_XDECREF(lxml); Py_XDECREF(version); - // Clear any errors that may have occurred - PyErr_Clear(); - return result; } From 06f6e46ff1600bae93965bad3ca8f75c52ab41d5 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 15:11:05 +0200 Subject: [PATCH 133/211] Add more manylinux distros to workflow (#301) --- .github/workflows/manylinux.yml | 9 ++++++++- setup.py | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 6d2dbb1a..f620ba44 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -1,13 +1,20 @@ name: manylinux on: [push, pull_request] jobs: - pep513: + manylinux: runs-on: ubuntu-latest + env: + # python-xmlsec is not compatible with xmlsec1 v1.3.3. + # So we need to use the rc until the next release. + # TODO: Remove it when xmlsec1 v1.3.4 is fully released. + PYXMLSEC_XMLSEC1_VERSION: "1.3.4-rc1" strategy: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2014_x86_64 + - manylinux_2_28_x86_64 + - musllinux_1_1_x86_64 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/setup.py b/setup.py index 277a8892..2d8347f7 100644 --- a/setup.py +++ b/setup.py @@ -238,7 +238,7 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1q') + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1t') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') @@ -391,7 +391,6 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-dependency-tracking', '--disable-shared', - '--enable-rebuild-docs=no', '--without-lzma', '--without-python', '--with-iconv={}'.format(self.prefix_dir), From d48d165f1555b81615f3e43e51ff2e5cd8e7d831 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 16:25:38 +0200 Subject: [PATCH 134/211] Fix pre-commit check (#302) --- tests/softhsm_setup.py | 108 +++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 59 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 432d4b1b..c2a805db 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -1,7 +1,7 @@ -""" -Testing the PKCS#11 shim layer. +"""Testing the PKCS#11 shim layer. + Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj -under licence "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +under license "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt """ import logging @@ -13,7 +13,7 @@ import unittest from typing import Dict, List, Optional, Tuple -DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') def paths_for_component(component: str, default_paths: List[str]): @@ -25,7 +25,7 @@ def find_alts(component_name, alts: List[str]) -> str: for a in alts: if os.path.exists(a): return a - raise unittest.SkipTest("Required component is missing: {}".format(component_name)) + raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: @@ -45,7 +45,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: conf = f.read() msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' msg = msg.format( - cmd=" ".join(args), + cmd=' '.join(args), code=rv, out=out.strip(), err=err.strip(), @@ -113,36 +113,34 @@ def _temp_dir() -> str: return d -@unittest.skipIf(component_path['P11_MODULE'] is None, "SoftHSM PKCS11 module not installed") +@unittest.skipIf(component_path['P11_MODULE'] is None, 'SoftHSM PKCS11 module not installed') def setup() -> None: - logging.debug("Creating test pkcs11 token using softhsm") + logging.debug('Creating test pkcs11 token using softhsm') try: global softhsm_conf softhsm_conf = _temp_file() - logging.debug("Generating softhsm.conf") - with open(softhsm_conf, "w") as f: + logging.debug('Generating softhsm.conf') + with open(softhsm_conf, 'w') as f: if softhsm_version == 2: softhsm_db = _temp_dir() f.write( - """ + f""" # Generated by test -directories.tokendir = %s +directories.tokendir = {softhsm_db} objectstore.backend = file log.level = DEBUG """ - % softhsm_db ) else: softhsm_db = _temp_file() f.write( - """ + f""" # Generated by test -0:%s +0:{softhsm_db} """ - % softhsm_db ) - logging.debug("Initializing the token") + logging.debug('Initializing the token') out, err = run_cmd( [ component_path['SOFTHSM'], @@ -159,18 +157,8 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - # logging.debug("Generating 1024 bit RSA key in token") - # run_cmd([component_path['PKCS11_TOOL'], - # '--module', component_path['P11_MODULE'], - # '-l', - # '-k', - # '--key-type', 'rsa:1024', - # '--id', 'a1b2', - # '--label', 'test', - # '--pin', 'secret1'], softhsm_conf=softhsm_conf) - hash_priv_key = _temp_file() - logging.debug("Converting test private key to format for softhsm") + logging.debug('Converting test private key to format for softhsm') run_cmd( [ component_path['OPENSSL'], @@ -189,7 +177,7 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - logging.debug("Importing the test key to softhsm") + logging.debug('Importing the test key to softhsm') run_cmd( [ component_path['SOFTHSM'], @@ -207,40 +195,42 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) run_cmd( - [component_path['PKCS11_TOOL'], '--module', component_path['P11_MODULE'], '-l', '--pin', 'secret1', '-O'], + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--pin', + 'secret1', + '-O', + ], softhsm_conf=softhsm_conf, ) signer_cert_pem = _temp_file() openssl_conf = _temp_file() - logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) - with open(openssl_conf, "w") as f: - # Might be needed with some versions of openssl, but in more recent versions dynamic_path breaks it. - # dynamic_path = ( - # "dynamic_path = %s" % component_path['P11_ENGINE'] - # if openssl_version.startswith(b'1.') - # else "" - # ) + logging.debug('Generating OpenSSL config for version %s', openssl_version) + with open(openssl_conf, 'w') as f: f.write( - "\n".join( + '\n'.join( [ - "openssl_conf = openssl_def", - "[openssl_def]", - "engines = engine_section", - "[engine_section]", - "pkcs11 = pkcs11_section", - "[req]", - "distinguished_name = req_distinguished_name", - "[req_distinguished_name]", - "[pkcs11_section]", - "engine_id = pkcs11", + 'openssl_conf = openssl_def', + '[openssl_def]', + 'engines = engine_section', + '[engine_section]', + 'pkcs11 = pkcs11_section', + '[req]', + 'distinguished_name = req_distinguished_name', + '[req_distinguished_name]', + '[pkcs11_section]', + 'engine_id = pkcs11', # dynamic_path, - "MODULE_PATH = %s" % component_path['P11_MODULE'], - "init = 0", + f"MODULE_PATH = {component_path['P11_MODULE']}", + 'init = 0', ] ) ) - with open(openssl_conf, "r") as f: + with open(openssl_conf, 'r') as f: logging.debug('-------- START DEBUG openssl_conf --------') logging.debug(f.readlines()) logging.debug('-------- END DEBUG openssl_conf --------') @@ -251,7 +241,7 @@ def setup() -> None: signer_cert_der = _temp_file() - logging.debug("Generating self-signed certificate") + logging.debug('Generating self-signed certificate') run_cmd( [ component_path['OPENSSL'], @@ -259,7 +249,7 @@ def setup() -> None: '-new', '-x509', '-subj', - "/CN=Test Signer", + '/CN=Test Signer', '-engine', 'pkcs11', '-config', @@ -292,7 +282,7 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - logging.debug("Importing certificate into token") + logging.debug('Importing certificate into token') run_cmd( [ @@ -316,15 +306,15 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - # TODO: Should be teardowned in teardown: + # TODO: Should be teardowned in teardown # noqa: T101 os.environ['SOFTHSM_CONF'] = softhsm_conf os.environ['SOFTHSM2_CONF'] = softhsm_conf except Exception as ex: - print("-" * 64) + print('-' * 64) traceback.print_exc() - print("-" * 64) - logging.error("PKCS11 tests disabled: unable to initialize test token: %s" % ex) + print('-' * 64) + logging.exception('PKCS11 tests disabled: unable to initialize test token') raise ex From 64d9bc1e2b23cffbf1dbcde23162b1ec3734e6a0 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 17:14:15 +0200 Subject: [PATCH 135/211] Remove unsupported Python 3.5 type hints (#303) Tuple, Dict, List and Optional not supported in Python 3.5, so we can't use them. Using them breaks MacOS workflows. --- tests/softhsm_setup.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index c2a805db..ef34b6c0 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -11,24 +11,23 @@ import tempfile import traceback import unittest -from typing import Dict, List, Optional, Tuple DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -def paths_for_component(component: str, default_paths: List[str]): +def paths_for_component(component: str, default_paths): env_path = os.environ.get(component) return [env_path] if env_path else default_paths -def find_alts(component_name, alts: List[str]) -> str: +def find_alts(component_name, alts) -> str: for a in alts: if os.path.exists(a): return a raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) -def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: +def run_cmd(args, softhsm_conf=None): env = {} if softhsm_conf is not None: env['SOFTHSM_CONF'] = softhsm_conf @@ -55,7 +54,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: return out, err -component_default_paths: Dict[str, List[str]] = { +component_default_paths = { 'P11_MODULE': [ '/usr/lib/softhsm/libsofthsm2.so', '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', @@ -85,7 +84,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: ], } -component_path: Dict[str, str] = { +component_path = { component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) for component_name, default_paths in component_default_paths.items() } @@ -96,9 +95,9 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() -p11_test_files: List[str] = [] -softhsm_conf: Optional[str] = None -softhsm_db: Optional[str] = None +p11_test_files = [] +softhsm_conf = None +softhsm_db = None def _temp_file() -> str: From 2a0438684f90979a7dc7c36fe4a7b31e30e688b7 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 17:45:41 +0200 Subject: [PATCH 136/211] Use old formatting to keep it compatible with Python 3.5 (#304) --- tests/softhsm_setup.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index ef34b6c0..247f1b18 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -123,20 +123,24 @@ def setup() -> None: if softhsm_version == 2: softhsm_db = _temp_dir() f.write( - f""" + """ # Generated by test -directories.tokendir = {softhsm_db} +directories.tokendir = {} objectstore.backend = file log.level = DEBUG -""" +""".format( + softhsm_db + ) ) else: softhsm_db = _temp_file() f.write( - f""" + """ # Generated by test -0:{softhsm_db} -""" +0:{} +""".format( + softhsm_db + ) ) logging.debug('Initializing the token') @@ -223,7 +227,7 @@ def setup() -> None: '[pkcs11_section]', 'engine_id = pkcs11', # dynamic_path, - f"MODULE_PATH = {component_path['P11_MODULE']}", + "MODULE_PATH = {}".format(component_path['P11_MODULE']), 'init = 0', ] ) From df37761539d0dd35ab428fbb4776e19591b2cc78 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 20:04:23 +0000 Subject: [PATCH 137/211] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 22.8.0 → 24.3.0](https://github.com/psf/black/compare/22.8.0...24.3.0) - [github.com/pre-commit/pre-commit-hooks: v4.3.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.3.0...v4.5.0) - [github.com/PyCQA/flake8: 5.0.4 → 7.0.0](https://github.com/PyCQA/flake8/compare/5.0.4...7.0.0) - [github.com/PyCQA/isort: 5.12.0 → 5.13.2](https://github.com/PyCQA/isort/compare/5.12.0...5.13.2) - [github.com/pre-commit/mirrors-mypy: v0.981 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v0.981...v1.9.0) - [github.com/pre-commit/pygrep-hooks: v1.9.0 → v1.10.0](https://github.com/pre-commit/pygrep-hooks/compare/v1.9.0...v1.10.0) --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf877f39..9678ec68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/psf/black - rev: 22.8.0 + rev: 24.3.0 hooks: - id: black types: [] files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace @@ -25,17 +25,17 @@ repos: - id: pretty-format-json args: [--autofix] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.0.0 hooks: - id: flake8 exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.981 + rev: v1.9.0 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) @@ -43,6 +43,6 @@ repos: files: ^.*.pyi?$ additional_dependencies: [lxml-stubs,types-docutils] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: rst-backticks From d5367145229dd316cc46f0df9e944dd28627bd56 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 20:04:38 +0000 Subject: [PATCH 138/211] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d8347f7..f0da1a2d 100644 --- a/setup.py +++ b/setup.py @@ -133,7 +133,7 @@ def run(self): [('MODULE_NAME', self.distribution.metadata.name), ('MODULE_VERSION', self.distribution.metadata.version)] ) # escape the XMLSEC_CRYPTO macro value, see mehcode/python-xmlsec#141 - for (key, value) in ext.define_macros: + for key, value in ext.define_macros: if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): ext.define_macros.remove((key, value)) ext.define_macros.append((key, '"{0}"'.format(value))) From 18f325019d4ac2ecfe32dba6b6815a58f6fd7988 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 5 Apr 2024 23:51:22 +0200 Subject: [PATCH 139/211] Fix opensuse-tumbleweed workflow (#305) --- .github/workflows/opensuse-tumbleweed.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index 273f7f7f..2f4caf49 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -11,9 +11,13 @@ jobs: - uses: actions/checkout@v1 - name: Install build dependencies run: | - zypper -n install -t pattern devel_basis + zypper refresh + zypper update + # The follwoing installs "devel_basis" pattern since installing the pattern fails because of few + # incompatibilty issues among packages + zypper -n install autoconf automake binutils bison cpp cpp13 flex gawk gcc gcc13 gdbm-devel gettext-runtime gettext-tools glibc-devel info kbd kbd-legacy libapparmor1 libasan8 libatomic1 libctf-nobfd0 libctf0 libdb-4_8 libfl-devel libfl2 libgdbm6 libgdbm_compat4 libgomp1 libhwasan0 libisl23 libitm1 libkmod2 liblsan0 libltdl7 libmpc3 libmpfr6 libseccomp2 libtextstyle0 libtool libtsan2 libubsan1 libxcrypt-devel libzio1 linux-glibc-devel m4 make makeinfo ncurses-devel pam-config patch perl perl-Text-Unidecode perl-base purge-kernels-service system-user-nobody systemd systemd-default-settings systemd-default-settings-branding-openSUSE systemd-presets-branding-openSUSE systemd-presets-common-SUSE tack update-alternatives zlib-devel PKGVER_NO_DOT=$(tr -d '.' <<< ${{ matrix.python-version }}) - zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${PKGVER_NO_DOT}-pip + zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${{ matrix.python-version }} -m venv .venv .venv/bin/python -m pip install --upgrade pip setuptools wheel - name: Build linux_x86_64 wheel @@ -22,7 +26,7 @@ jobs: rm -rf build/ - name: Install test dependencies run: | - .venv/bin/python -m pip install --upgrade -r requirements-test.txt + .venv/bin/python -m pip install --upgrade --no-binary=lxml -r requirements-test.txt .venv/bin/python -m pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - name: Run tests run: | From 83ee6324c6e1aaeafef96a43a6584e3991fa2272 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 20:18:14 -0300 Subject: [PATCH 140/211] Fetch latest openssl version, when doing static build. --- .github/workflows/manylinux.yml | 6 ++- setup.py | 83 +++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index f620ba44..ea05780b 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -18,11 +18,15 @@ jobs: container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 - - name: Install build dependencies + - name: Install python build dependencies run: | # https://github.com/actions/runner/issues/2033 chown -R $(id -u):$(id -g) $PWD /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build + - name: Install system build dependencies (manylinux) + run: | + yum install -y perl-core + if: contains(matrix.image, 'manylinux') - name: Set environment variables shell: bash run: | diff --git a/setup.py b/setup.py index f0da1a2d..32c4d134 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,8 @@ from distutils.errors import DistutilsError from distutils.version import StrictVersion as Version from pathlib import Path -from urllib.request import urlcleanup, urljoin, urlopen, urlretrieve +from urllib.parse import urljoin +from urllib.request import urlcleanup, urlopen, urlretrieve, Request from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig @@ -31,31 +32,60 @@ def handle_starttag(self, tag, attrs): self.hrefs.append(value) +def make_request(url, github_token=None, json_response=False): + headers = {'User-Agent': 'https://github.com/xmlsec/python-xmlsec'} + if github_token: + headers['authorization'] = "Bearer " + github_token + request = Request(url, headers=headers) + with contextlib.closing(urlopen(request)) as r: + if json_response: + return json.load(r) + else: + charset = r.headers.get_content_charset() or 'utf-8' + content = r.read().decode(charset) + return content + + def latest_release_from_html(url, matcher): - with contextlib.closing(urlopen(url)) as r: - charset = r.headers.get_content_charset() or 'utf-8' - content = r.read().decode(charset) - collector = HrefCollector() - collector.feed(content) - hrefs = collector.hrefs - - def comp(text): - try: - return Version(matcher.match(text).groupdict()['version']) - except (AttributeError, ValueError): - return Version('0.0') + content = make_request(url) + collector = HrefCollector() + collector.feed(content) + hrefs = collector.hrefs + + def comp(text): + try: + return Version(matcher.match(text).groupdict()['version']) + except (AttributeError, ValueError): + return Version('0.0') - latest = max(hrefs, key=comp) - return '{}/{}'.format(url, latest) + latest = max(hrefs, key=comp) + return '{}/{}'.format(url, latest) def latest_release_from_gnome_org_cache(url, lib_name): cache_url = '{}/cache.json'.format(url) - with contextlib.closing(urlopen(cache_url)) as r: - cache = json.load(r) - latest_version = cache[2][lib_name][-1] - latest_source = cache[1][lib_name][latest_version]['tar.xz'] - return '{}/{}'.format(url, latest_source) + cache = make_request(cache_url, json_response=True) + latest_version = cache[2][lib_name][-1] + latest_source = cache[1][lib_name][latest_version]['tar.xz'] + return '{}/{}'.format(url, latest_source) + + +def latest_release_from_github_api(repo): + api_url = 'https://api.github.com/repos/{}/releases'.format(repo) + + # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited + token = os.environ.get("GH_TOKEN") + if token: + log.info("Using GitHub token to avoid rate limiting") + api_releases = make_request(api_url, token, json_response=True) + releases = [r['tarball_url'] for r in api_releases if r['prerelease'] is False and r['draft'] is False] + if not releases: + raise DistutilsError('No release found for {}'.format(repo)) + return releases[0] + + +def latest_openssl_release(): + return latest_release_from_github_api('openssl/openssl') def latest_zlib_release(): @@ -238,7 +268,7 @@ def prepare_static_build_win(self): ext.include_dirs = [str(p.absolute()) for p in includes] def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1t') + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') @@ -250,8 +280,13 @@ def prepare_static_build_linux(self): if openssl_tar is None: self.info('{:10}: {}'.format('OpenSSL', 'source tar not found, downloading ...')) openssl_tar = self.libs_dir / 'openssl.tar.gz' - self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - urlretrieve('https://www.openssl.org/source/openssl-{}.tar.gz'.format(self.openssl_version), str(openssl_tar)) + if self.openssl_version is None: + url = latest_openssl_release() + self.info('{:10}: {}'.format('OpenSSL', 'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://api.github.com/repos/openssl/openssl/tarball/openssl-{}'.format(self.openssl_version) + self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) + urlretrieve(url, str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) @@ -361,7 +396,7 @@ def prepare_static_build_linux(self): self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC'], cwd=str(openssl_dir), env=env) + subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'], cwd=str(openssl_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) subprocess.check_output( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env From 73cca3cabf203b6660ce3396621047789bf77d94 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:23:29 +0000 Subject: [PATCH 141/211] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32c4d134..cdf95c41 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from distutils.version import StrictVersion as Version from pathlib import Path from urllib.parse import urljoin -from urllib.request import urlcleanup, urlopen, urlretrieve, Request +from urllib.request import Request, urlcleanup, urlopen, urlretrieve from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig From a51d1ff1ef0b1bb98df97c91bfc0f4504392ff9c Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:02:39 -0300 Subject: [PATCH 142/211] Allow static build from MacOS --- setup.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index cdf95c41..7b5d43d6 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,17 @@ def latest_xmlsec_release(): return latest_release_from_html('https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz')) +class CrossCompileInfo: + def __init__(self, host, arch, compiler): + self.host = host + self.arch = arch + self.compiler = compiler + + @property + def triplet(self): + return "{}-{}-{}".format(self.host, self.arch, self.compiler) + + class build_ext(build_ext_orig): def info(self, message): self.announce(message, level=log.INFO) @@ -136,7 +147,9 @@ def run(self): if sys.platform == 'win32': self.prepare_static_build_win() elif 'linux' in sys.platform: - self.prepare_static_build_linux() + self.prepare_static_build(sys.platform) + elif 'darwin' in sys.platform: + self.prepare_static_build(sys.platform) else: import pkgconfig @@ -267,7 +280,7 @@ def prepare_static_build_win(self): includes.append(next(p / 'xmlsec' for p in includes if (p / 'xmlsec').is_dir())) ext.include_dirs = [str(p.absolute()) for p in includes] - def prepare_static_build_linux(self): + def prepare_static_build(self, build_platform): self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') @@ -387,16 +400,42 @@ def prepare_static_build_linux(self): prefix_arg = '--prefix={}'.format(self.prefix_dir) - cflags = ['-fPIC'] env = os.environ.copy() - if 'CFLAGS' in env: - env['CFLAGS'].append(' '.join(cflags)) - else: - env['CFLAGS'] = ' '.join(cflags) + cflags = [] + if env.get('CFLAGS'): + cflags.append(env['CFLAGS']) + cflags.append('-fPIC') + ldflags = [] + if env.get('LDFLAGS'): + ldflags.append(env['LDFLAGS']) + + cross_compiling = False + if build_platform == 'darwin': + import platform + + arch = self.plat_name.rsplit('-', 1)[1] + if arch != platform.machine() and arch in ('x86_64', 'arm64'): + self.info('Cross-compiling for {}'.format(arch)) + cflags.append('-arch {}'.format(arch)) + ldflags.append('-arch {}'.format(arch)) + cross_compiling = CrossCompileInfo('darwin64', arch, 'cc') + major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11: + if 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = "11.0" + + env['CFLAGS'] = ' '.join(cflags) + env['LDFLAGS'] = ' '.join(ldflags) self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'], cwd=str(openssl_dir), env=env) + openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if cross_compiling: + openssl_config_cmd.insert(0, './Configure') + openssl_config_cmd.append(cross_compiling.triplet) + else: + openssl_config_cmd.insert(0, './config') + subprocess.check_output(openssl_config_cmd, cwd=str(openssl_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) subprocess.check_output( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env @@ -408,10 +447,22 @@ def prepare_static_build_linux(self): subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) + host_arg = "" + if cross_compiling: + host_arg = '--host={}'.format(cross_compiling.arch) + self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) subprocess.check_output( - ['./configure', prefix_arg, '--disable-dependency-tracking', '--disable-shared'], cwd=str(libiconv_dir), env=env + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + host_arg, + ], + cwd=str(libiconv_dir), + env=env, ) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) subprocess.check_output( @@ -430,6 +481,7 @@ def prepare_static_build_linux(self): '--without-python', '--with-iconv={}'.format(self.prefix_dir), '--with-zlib={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxml2_dir), env=env, @@ -450,6 +502,7 @@ def prepare_static_build_linux(self): '--without-python', '--without-crypto', '--with-libxml-prefix={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxslt_dir), env=env, @@ -460,10 +513,8 @@ def prepare_static_build_linux(self): ) self.info('Building xmlsec1') - if 'LDFLAGS' in env: - env['LDFLAGS'].append(' -lpthread') - else: - env['LDFLAGS'] = '-lpthread' + ldflags.append('-lpthread') + env['LDFLAGS'] = ' '.join(ldflags) xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) subprocess.check_output( [ @@ -480,6 +531,7 @@ def prepare_static_build_linux(self): '--with-openssl={}'.format(self.prefix_dir), '--with-libxml={}'.format(self.prefix_dir), '--with-libxslt={}'.format(self.prefix_dir), + host_arg, ], cwd=str(xmlsec1_dir), env=env, @@ -517,7 +569,8 @@ def prepare_static_build_linux(self): ext.include_dirs.extend([str(p.absolute()) for p in (self.prefix_dir / 'include').iterdir() if p.is_dir()]) ext.library_dirs = [] - ext.libraries = ['m', 'rt'] + if build_platform == 'linux': + ext.libraries = ['m', 'rt'] extra_objects = [ 'libxmlsec1.a', 'libxslt.a', From 47e711b0317955f0f363985606f0f9fdf6939bea Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 7 Apr 2024 19:13:33 -0300 Subject: [PATCH 143/211] Add CI action for OSX static build --- .github/workflows/macosx.yml | 3 +++ setup.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index c477157c..5baa1e44 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -6,6 +6,7 @@ jobs: strategy: matrix: python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + static_deps: ["static", ""] steps: - uses: actions/checkout@v3 - name: Setup Python @@ -21,6 +22,7 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + PYXMLSEC_STATIC_DEPS: ${{ matrix.static_deps }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build @@ -43,3 +45,4 @@ jobs: /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt + if: matrix.static_deps != 'static' diff --git a/setup.py b/setup.py index 7b5d43d6..7e9e8ba1 100644 --- a/setup.py +++ b/setup.py @@ -38,11 +38,11 @@ def make_request(url, github_token=None, json_response=False): headers['authorization'] = "Bearer " + github_token request = Request(url, headers=headers) with contextlib.closing(urlopen(request)) as r: + charset = r.headers.get_content_charset() or 'utf-8' + content = r.read().decode(charset) if json_response: - return json.load(r) + return json.loads(content) else: - charset = r.headers.get_content_charset() or 'utf-8' - content = r.read().decode(charset) return content From 6922ccba43f544b90aaa782dfd7f8ff4f51a1103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 20:18:07 +0000 Subject: [PATCH 144/211] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9678ec68..820778ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace From e9251d34b735caac7160735872d5deeb7910285d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sun, 7 Apr 2024 21:56:55 -0300 Subject: [PATCH 145/211] Fix linux brew workflow --- .github/workflows/linuxbrew.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 191e2001..886bd8c9 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -3,26 +3,28 @@ on: [push, pull_request] jobs: linuxbrew: runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Install build dependencies + - name: Install brew run: | sudo apt install -y build-essential procps curl file git /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - test -d ~/.linuxbrew && eval $(~/.linuxbrew/bin/brew shellenv) - test -r ~/.bash_profile && echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.bash_profile - echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile + echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH + - name: Install build dependencies + run: | brew update - brew install python gcc libxml2 libxmlsec1 pkg-config + brew install python@${{ matrix.python }} gcc libxml2 libxmlsec1 pkg-config + echo "/home/linuxbrew/.linuxbrew/opt/python@${{ matrix.python }}/libexec/bin" >> $GITHUB_PATH + - name: Install python dependencies + run: | pip3 install --upgrade setuptools wheel build - ln -s $(brew --prefix)/bin/gcc-12 $(brew --prefix)/bin/gcc-5 - ls -l $(brew --prefix)/bin/gcc* - name: Build linux_x86_64 wheel run: | + export CFLAGS="-I$(brew --prefix)/include" + export LDFLAGS="-L$(brew --prefix)/lib" python3 -m build rm -rf build/ - name: Install test dependencies From e789da55100ab1ce96d2f9d0d798d5a67b16b74d Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 8 Apr 2024 21:59:40 -0300 Subject: [PATCH 146/211] Add gh_token to avoid getting rate limited --- .github/workflows/macosx.yml | 1 + .github/workflows/manylinux.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 5baa1e44..6d0548e8 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -23,6 +23,7 @@ jobs: CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" PYXMLSEC_STATIC_DEPS: ${{ matrix.static_deps }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index ea05780b..a80ad67b 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -34,6 +34,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build - name: Label manylinux wheel From b864c31c2c887e7ddf4fdc6c9faa2bfd20ef6d64 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 20 Mar 2024 21:12:14 -0300 Subject: [PATCH 147/211] Use ci build wheel to build staticly linked wheels for linux and OSX --- .github/workflows/wheels.yml | 122 +++++++++++++++++++++++++++++++++++ pyproject.toml | 28 ++++++++ 2 files changed, 150 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000..0d83f59b --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,122 @@ +name: Wheel build + +on: + release: + types: [created] + schedule: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + - cron: "42 3 * * 4" + push: + pull_request: + workflow_dispatch: + +permissions: {} + +jobs: + sdist: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: "3.x" + + - name: Install build dependencies + run: | + pip install --upgrade pip setuptools wheel + + - name: Package source dist + run: python setup.py sdist + + - name: Install test dependencies + run: | + sudo apt-get update -y -q + sudo apt-get install -y -q libxml2-dev libxslt1-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl + pip install --upgrade -r requirements-test.txt --no-binary lxml + pip install dist/xmlsec-$(python setup.py --version).tar.gz + + - name: Run tests + run: pytest -v --color=yes + + - name: Upload sdist + uses: actions/upload-artifact@v4.3.1 + with: + name: sdist + path: dist/*.tar.gz + + generate-wheels-matrix: + # Create a matrix of all architectures & versions to build. + # This enables the next step to run cibuildwheel in parallel. + # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 + name: Generate wheels matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v4 + - name: Install cibuildwheel + # Nb. keep cibuildwheel version pin consistent with job below + run: pipx install cibuildwheel==2.16.5 + - id: set-matrix + # Once we have the windows build figured out, it can be added here + # by updating the matrix to include windows builds as well. + # See example here: + # https://github.com/lxml/lxml/blob/3ccc7d583e325ceb0ebdf8fc295bbb7fc8cd404d/.github/workflows/wheels.yml#L95C1-L106C51 + run: | + MATRIX=$( + { + cibuildwheel --print-build-identifiers --platform linux \ + | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos \ + | jq -nRc '{"only": inputs, "os": "macos-latest"}' + } | jq -sc + ) + echo "include=$MATRIX" + echo "include=$MATRIX" >> $GITHUB_OUTPUT + + build_wheels: + name: Build for ${{ matrix.only }} + needs: generate-wheels-matrix + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + with: + only: ${{ matrix.only }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/upload-artifact@v4.3.1 + with: + path: ./wheelhouse/*.whl + name: xmlsec-wheel-${{ matrix.only }} diff --git a/pyproject.toml b/pyproject.toml index e28878e3..f3ddbdbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,3 +45,31 @@ known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] [build-system] requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] + +[tool.cibuildwheel] +build-verbosity = 1 +build-frontend = "build" +skip = ["pp*", "*-musllinux_i686"] +test-command = "pytest -v --color=yes {package}/tests" +before-test = "pip install -r requirements-test.txt" +test-skip = "*-macosx_arm64" + +[tool.cibuildwheel.environment] +PYXMLSEC_STATIC_DEPS = "true" + +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64", "i686"] +environment-pass = [ + "PYXMLSEC_LIBXML2_VERSION", + "PYXMLSEC_LIBXSLT_VERSION", + "PYXMLSEC_STATIC_DEPS", + "GH_TOKEN" +] + +[tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] +before-all = "brew install perl" + +[[tool.cibuildwheel.overrides]] +select = "*-manylinux*" +before-all = "yum install -y perl-core" From 6741ff80cf0df8c8e9e611baaf77f4c2c68321a1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Tue, 9 Apr 2024 16:35:03 -0300 Subject: [PATCH 148/211] Make skipped wheel builds match upstream lxml. --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f3ddbdbb..99927b4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,16 @@ requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>= [tool.cibuildwheel] build-verbosity = 1 build-frontend = "build" -skip = ["pp*", "*-musllinux_i686"] +skip = [ + "pp*", + "*-musllinux_i686", + # LXML doesn't publish wheels for these platforms, which makes it + # difficult for us to build wheels, so we exclude them. + "cp36-manylinux_aarch64", + "cp37-manylinux_aarch64", + "cp36-musllinux_aarch64", + "cp37-musllinux_aarch64", +] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" test-skip = "*-macosx_arm64" From 3edf947c05c5e853dc77cc66c8dcc0b19735c033 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Tue, 9 Apr 2024 16:47:00 -0300 Subject: [PATCH 149/211] Remove PYXMLSEC_XMLSEC1_VERSION override. --- .github/workflows/manylinux.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index a80ad67b..a44776b3 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -3,11 +3,6 @@ on: [push, pull_request] jobs: manylinux: runs-on: ubuntu-latest - env: - # python-xmlsec is not compatible with xmlsec1 v1.3.3. - # So we need to use the rc until the next release. - # TODO: Remove it when xmlsec1 v1.3.4 is fully released. - PYXMLSEC_XMLSEC1_VERSION: "1.3.4-rc1" strategy: matrix: python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] From 1b3b527fd27ea0f48ee3c69354249d90d56cec23 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Wed, 17 Apr 2024 18:16:58 +0200 Subject: [PATCH 150/211] Add windows wheel build (#313) --- .github/workflows/wheels.yml | 4 +++- pyproject.toml | 3 +++ setup.py | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0d83f59b..ccd62cf8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -81,7 +81,9 @@ jobs: cibuildwheel --print-build-identifiers --platform linux \ | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ && cibuildwheel --print-build-identifiers --platform macos \ - | jq -nRc '{"only": inputs, "os": "macos-latest"}' + | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows \ + | jq -nRc '{"only": inputs, "os": "windows-2019"}' } | jq -sc ) echo "include=$MATRIX" diff --git a/pyproject.toml b/pyproject.toml index 99927b4b..9b6469d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,9 @@ environment-pass = [ archs = ["x86_64", "arm64"] before-all = "brew install perl" +[tool.cibuildwheel.windows] +archs = ["AMD64", "x86"] + [[tool.cibuildwheel.overrides]] select = "*-manylinux*" before-all = "yum install -y perl-core" diff --git a/setup.py b/setup.py index 7e9e8ba1..92588ebc 100644 --- a/setup.py +++ b/setup.py @@ -213,19 +213,19 @@ def run(self): super(build_ext, self).run() def prepare_static_build_win(self): - release_url = 'https://github.com/bgaifullin/libxml2-win-binaries/releases/download/v2018.08/' - if sys.maxsize > 2147483647: + release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2024.04.17/' + if sys.maxsize > 2147483647: # 2.0 GiB suffix = 'win64' else: suffix = 'win32' libs = [ - 'libxml2-2.9.4.{}.zip'.format(suffix), - 'libxslt-1.1.29.{}.zip'.format(suffix), - 'zlib-1.2.8.{}.zip'.format(suffix), - 'iconv-1.14.{}.zip'.format(suffix), - 'openssl-1.0.1.{}.zip'.format(suffix), - 'xmlsec-1.2.24.{}.zip'.format(suffix), + 'libxml2-2.11.7.{}.zip'.format(suffix), + 'libxslt-1.1.37.{}.zip'.format(suffix), + 'zlib-1.2.12.{}.zip'.format(suffix), + 'iconv-1.16-1.{}.zip'.format(suffix), + 'openssl-3.0.8.{}.zip'.format(suffix), + 'xmlsec-1.3.4.{}.zip'.format(suffix), ] for libfile in libs: @@ -262,7 +262,7 @@ def prepare_static_build_win(self): ext.libraries = [ 'libxmlsec_a', 'libxmlsec-openssl_a', - 'libeay32', + 'libcrypto', 'iconv_a', 'libxslt_a', 'libexslt_a', @@ -599,6 +599,7 @@ def prepare_static_build(self, build_platform): use_scm_version=True, description='Python bindings for the XML Security Library', long_description=long_desc, + long_description_content_type='text/markdown', ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, python_requires='>=3.5', From 67cd4ac73e4fceac4b4eb6a320067cad33f79213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 19 Jun 2024 17:43:07 +0200 Subject: [PATCH 151/211] Explicitly cast the pointer type in PyXmlSec_ClearReplacedNodes Fixes https://github.com/xmlsec/python-xmlsec/issues/323 --- src/enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/enc.c b/src/enc.c index 5453ef99..c2bc94bf 100644 --- a/src/enc.c +++ b/src/enc.c @@ -204,7 +204,7 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume PYXMLSEC_DEBUGF("clear replaced node %p", n); nn = n->next; // if n has references, it will not be deleted - elem = PyXmlSec_elementFactory(doc, n); + elem = (PyXmlSec_LxmlElementPtr*)PyXmlSec_elementFactory(doc, n); if (NULL == elem) xmlFreeNode(n); else From e2ca04f26a0679db01feb0687d6ae93542bcb47f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 30 Oct 2024 22:03:02 -0300 Subject: [PATCH 152/211] Build wheel for python 3.13 --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ccd62cf8..c3aaf48d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -69,7 +69,7 @@ jobs: - uses: actions/checkout@v4 - name: Install cibuildwheel # Nb. keep cibuildwheel version pin consistent with job below - run: pipx install cibuildwheel==2.16.5 + run: pipx install cibuildwheel==2.21.3 - id: set-matrix # Once we have the windows build figured out, it can be added here # by updating the matrix to include windows builds as well. @@ -112,7 +112,7 @@ jobs: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.21.3 with: only: ${{ matrix.only }} env: From a2050130537cc31cae1806a8a3663b4927bab111 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Wed, 30 Oct 2024 22:05:44 -0300 Subject: [PATCH 153/211] Trigger build From 08b3e914ff2acaa3288313de15374355b76ec2d2 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 09:10:54 -0300 Subject: [PATCH 154/211] Bump library versions --- .github/workflows/wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c3aaf48d..85410d98 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -99,6 +99,10 @@ jobs: matrix: include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} + env: + PYXMLSEC_LIBXML2_VERSION: 2.12.9 + PYXMLSEC_LIBXSLT_VERSION: 1.1.42 + steps: - name: Check out the repo uses: actions/checkout@v4 From 6a2769179429c17263d45b15e0a53cd05eca9c71 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 09:37:23 -0300 Subject: [PATCH 155/211] Fix sdist workflow --- .github/workflows/sdist.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index e7c0f39d..27985647 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -3,12 +3,15 @@ on: [push, pull_request] jobs: sdist: runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: ${{ matrix.python }} - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel From 9e134cbb915267ebd86b8b876b9f75287be13f75 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 09:39:10 -0300 Subject: [PATCH 156/211] Add missing apt-get update --- .github/workflows/sdist.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 27985647..e987cdce 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -20,6 +20,7 @@ jobs: python setup.py sdist - name: Install test dependencies run: | + sudo apt-get update sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl pip install --upgrade -r requirements-test.txt --no-binary lxml pip install dist/xmlsec-$(python setup.py --version).tar.gz From b5847db453a1a7780aa18f5bcb5cc28811a70ed7 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 09:48:06 -0300 Subject: [PATCH 157/211] Fix linux brew workflow --- .github/workflows/linuxbrew.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 886bd8c9..1fd2358d 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 - name: Install brew From 7216b770b6bfc6663bae503c74be3a4108828e05 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 10:00:56 -0300 Subject: [PATCH 158/211] Install in venv --- .github/workflows/linuxbrew.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 1fd2358d..14acfe94 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -18,19 +18,19 @@ jobs: brew update brew install python@${{ matrix.python }} gcc libxml2 libxmlsec1 pkg-config echo "/home/linuxbrew/.linuxbrew/opt/python@${{ matrix.python }}/libexec/bin" >> $GITHUB_PATH - - name: Install python dependencies + - name: Build wheel run: | + python3 -m venv build_venv + source build_venv/bin/activate pip3 install --upgrade setuptools wheel build - - name: Build linux_x86_64 wheel - run: | export CFLAGS="-I$(brew --prefix)/include" export LDFLAGS="-L$(brew --prefix)/lib" python3 -m build rm -rf build/ - - name: Install test dependencies + - name: Run tests run: | + python3 -m venv test_venv + source test_venv/bin/activate pip3 install --upgrade -r requirements-test.txt pip3 install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - - name: Run tests - run: | pytest -v --color=yes From f0fa15f3c17c6135bf060e3246e6abe21ae28b1a Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 10:07:40 -0300 Subject: [PATCH 159/211] Make sure we build lxml --- .github/workflows/linuxbrew.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 14acfe94..d8996fd6 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -31,6 +31,6 @@ jobs: run: | python3 -m venv test_venv source test_venv/bin/activate - pip3 install --upgrade -r requirements-test.txt + pip3 install --upgrade --no-binary=lxml -r requirements-test.txt pip3 install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ pytest -v --color=yes From ba1e39c1b02a2fd954c2e8a34ed0b9bd4b69e35f Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 10:22:06 -0300 Subject: [PATCH 160/211] Fix osx workflow --- .github/workflows/macosx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 6d0548e8..21be9b75 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] static_deps: ["static", ""] steps: - uses: actions/checkout@v3 From 911c537d12ca0296daf89b90513b2efb95b60b6a Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 10:29:12 -0300 Subject: [PATCH 161/211] Build lxml with --no-binary --- .github/workflows/macosx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 21be9b75..1b82d22d 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -35,7 +35,8 @@ jobs: echo "LLVM_PROFILE_FILE=pyxmlsec.profraw" >> $GITHUB_ENV - name: Install test dependencies run: | - pip install coverage --upgrade -r requirements-test.txt + export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" + pip install coverage --upgrade --no-binary=lxml -r requirements-test.txt pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ echo "PYXMLSEC_LIBFILE=$(python -c 'import xmlsec; print(xmlsec.__file__)')" >> $GITHUB_ENV - name: Run tests From 413c1af090178b20f7208ddceed1d0a083c336ca Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Thu, 31 Oct 2024 11:00:34 -0300 Subject: [PATCH 162/211] Ignore macos platforms that lxml no longer supports --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9b6469d4..8c03f910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,9 @@ skip = [ "cp37-manylinux_aarch64", "cp36-musllinux_aarch64", "cp37-musllinux_aarch64", + "cp36-macosx*", + "cp37-macosx*", + "cp38-macosx*", ] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" From e634b2ff5bdfd1ca6dec23f11d6d689e818aa0fa Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Tue, 5 Nov 2024 13:53:31 -0500 Subject: [PATCH 163/211] Update: PR#333 https://github.com/xmlsec/python-xmlsec/pull/333 --- .github/workflows/opensuse-tumbleweed.yml | 33 ----------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/opensuse-tumbleweed.yml diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml deleted file mode 100644 index 2f4caf49..00000000 --- a/.github/workflows/opensuse-tumbleweed.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: opensuse-tumbleweed -on: [push, pull_request] -jobs: - tumbleweed: - runs-on: ubuntu-latest - container: opensuse/tumbleweed - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - steps: - - uses: actions/checkout@v1 - - name: Install build dependencies - run: | - zypper refresh - zypper update - # The follwoing installs "devel_basis" pattern since installing the pattern fails because of few - # incompatibilty issues among packages - zypper -n install autoconf automake binutils bison cpp cpp13 flex gawk gcc gcc13 gdbm-devel gettext-runtime gettext-tools glibc-devel info kbd kbd-legacy libapparmor1 libasan8 libatomic1 libctf-nobfd0 libctf0 libdb-4_8 libfl-devel libfl2 libgdbm6 libgdbm_compat4 libgomp1 libhwasan0 libisl23 libitm1 libkmod2 liblsan0 libltdl7 libmpc3 libmpfr6 libseccomp2 libtextstyle0 libtool libtsan2 libubsan1 libxcrypt-devel libzio1 linux-glibc-devel m4 make makeinfo ncurses-devel pam-config patch perl perl-Text-Unidecode perl-base purge-kernels-service system-user-nobody systemd systemd-default-settings systemd-default-settings-branding-openSUSE systemd-presets-branding-openSUSE systemd-presets-common-SUSE tack update-alternatives zlib-devel - PKGVER_NO_DOT=$(tr -d '.' <<< ${{ matrix.python-version }}) - zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel - python${{ matrix.python-version }} -m venv .venv - .venv/bin/python -m pip install --upgrade pip setuptools wheel - - name: Build linux_x86_64 wheel - run: | - .venv/bin/python setup.py bdist_wheel - rm -rf build/ - - name: Install test dependencies - run: | - .venv/bin/python -m pip install --upgrade --no-binary=lxml -r requirements-test.txt - .venv/bin/python -m pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - - name: Run tests - run: | - .venv/bin/python -m pytest -v --color=yes From 1104b4cb56078d7753bad9b09428e889ed01f37f Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Tue, 5 Nov 2024 16:12:06 -0500 Subject: [PATCH 164/211] Also support Macports --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 60fb3ea1..bd3a1be9 100644 --- a/README.rst +++ b/README.rst @@ -93,6 +93,12 @@ Mac brew install libxml2 libxmlsec1 pkg-config +or + +.. code-block:: bash + + port install libxml2 xmlsec pkgconfig + Alpine ^^^^^^ From 79e7151c879437a0990d22e08b29bf6438a69691 Mon Sep 17 00:00:00 2001 From: Yusuke Hayashi Date: Mon, 10 Feb 2025 18:17:35 +0900 Subject: [PATCH 165/211] Update enc.c --- src/enc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/enc.c b/src/enc.c index c2bc94bf..42195dd3 100644 --- a/src/enc.c +++ b/src/enc.c @@ -195,7 +195,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec // release the replaced nodes in a way safe for `lxml` static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocumentPtr doc) { - PyXmlSec_LxmlElementPtr* elem; + PyXmlSec_LxmlElementPtr elem; // release the replaced nodes in a way safe for `lxml` xmlNodePtr n = ctx->replacedNodeList; xmlNodePtr nn; @@ -204,7 +204,7 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume PYXMLSEC_DEBUGF("clear replaced node %p", n); nn = n->next; // if n has references, it will not be deleted - elem = (PyXmlSec_LxmlElementPtr*)PyXmlSec_elementFactory(doc, n); + elem = (PyXmlSec_LxmlElementPtr)PyXmlSec_elementFactory(doc, n); if (NULL == elem) xmlFreeNode(n); else From 6498eac71c8074b2311dd3a7e90060260d0bd4c9 Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Tue, 11 Mar 2025 15:47:34 -0400 Subject: [PATCH 166/211] Update w/ latest rev of ext libs --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 85410d98..af3fb57d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -100,7 +100,7 @@ jobs: include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} env: - PYXMLSEC_LIBXML2_VERSION: 2.12.9 + PYXMLSEC_LIBXML2_VERSION: 2.13.5 PYXMLSEC_LIBXSLT_VERSION: 1.1.42 steps: From 967754b6ec084d3ee5baa73ad256cf19f0f231de Mon Sep 17 00:00:00 2001 From: Jim Jagielski Date: Fri, 11 Apr 2025 11:49:01 -0400 Subject: [PATCH 167/211] Ensure we don't use the wheel, which uses older versions of lxml2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 014bfe10..3490b822 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -lxml >= 3.8.0, !=4.7.0 +lxml >= 3.8.0, !=4.7.0 --no-binary=lxml From f3f9ba34af99f8c857f9c4f1fe95f8485a0e077f Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Sat, 5 Jul 2025 16:15:16 +0200 Subject: [PATCH 168/211] Revert "Ensure we don't use the wheel, which uses older versions of lxml2" This reverts commit 967754b6ec084d3ee5baa73ad256cf19f0f231de. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3490b822..014bfe10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -lxml >= 3.8.0, !=4.7.0 --no-binary=lxml +lxml >= 3.8.0, !=4.7.0 From 40b755f700b413684abca2d71d78a14c9740cfc7 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Sat, 5 Jul 2025 16:16:24 +0200 Subject: [PATCH 169/211] Update library versions based on lxml 6.0.0 --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index af3fb57d..4303504a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -100,8 +100,8 @@ jobs: include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} env: - PYXMLSEC_LIBXML2_VERSION: 2.13.5 - PYXMLSEC_LIBXSLT_VERSION: 1.1.42 + PYXMLSEC_LIBXML2_VERSION: 2.14.4 + PYXMLSEC_LIBXSLT_VERSION: 1.1.43 steps: - name: Check out the repo From 6e691b36f95b9421ccacf64d6c5ec269bc0a9e43 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Sat, 5 Jul 2025 16:31:28 +0200 Subject: [PATCH 170/211] Enable ripemd60 As suggested in https://github.com/xmlsec/python-xmlsec/issues/344#issuecomment-2894393505 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 92588ebc..c1cdce44 100644 --- a/setup.py +++ b/setup.py @@ -523,6 +523,7 @@ def prepare_static_build(self, build_platform): '--disable-shared', '--disable-gost', '--enable-md5', + '--enable-ripemd160', '--disable-crypto-dl', '--enable-static=yes', '--enable-shared=no', From d6ea8dec42a870a3971b15e83d65c5c08d5664ac Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 00:46:52 +0200 Subject: [PATCH 171/211] Fix failing tests in sdist workflows --- .github/workflows/sdist.yml | 6 +++++- .github/workflows/wheels.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index e987cdce..04f22c14 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -2,7 +2,11 @@ name: sdist on: [push, pull_request] jobs: sdist: - runs-on: ubuntu-latest + # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev + # v1.2.39, which has a bug that causes tests/test_pkcs11.py to fail. + # (It thinks the softhsm engine has a public key instead of a private key.) + # libxmlsec1 <=1.2.33 or >=1.2.42 works. TODO: Try 26.04 when available. + runs-on: ubuntu-22.04 strategy: matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13"] diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4303504a..f9294fd3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -19,7 +19,11 @@ permissions: {} jobs: sdist: - runs-on: ubuntu-latest + # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev + # v1.2.39, which has a bug that causes tests/test_pkcs11.py to fail. + # (It thinks the softhsm engine has a public key instead of a private key.) + # libxmlsec1 <=1.2.33 or >=1.2.42 works. TODO: Try 26.04 when available. + runs-on: ubuntu-22.04 permissions: contents: write From bcd364cdb8e99f49172eaae67b2572ffaa7b4e61 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:09:15 +0200 Subject: [PATCH 172/211] Show stdout in setup.py, not only stderr --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index c1cdce44..d5cf53aa 100644 --- a/setup.py +++ b/setup.py @@ -435,17 +435,17 @@ def prepare_static_build(self, build_platform): openssl_config_cmd.append(cross_compiling.triplet) else: openssl_config_cmd.insert(0, './config') - subprocess.check_output(openssl_config_cmd, cwd=str(openssl_dir), env=env) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) - subprocess.check_output( + subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env ) self.info('Building zlib') zlib_dir = next(self.build_libs_dir.glob('zlib-*')) - subprocess.check_output(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) + subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) host_arg = "" if cross_compiling: @@ -453,7 +453,7 @@ def prepare_static_build(self, build_platform): self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) - subprocess.check_output( + subprocess.check_call( [ './configure', prefix_arg, @@ -464,14 +464,14 @@ def prepare_static_build(self, build_platform): cwd=str(libiconv_dir), env=env, ) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) - subprocess.check_output( + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libiconv_dir), env=env ) self.info('Building LibXML2') libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) - subprocess.check_output( + subprocess.check_call( [ './configure', prefix_arg, @@ -486,14 +486,14 @@ def prepare_static_build(self, build_platform): cwd=str(libxml2_dir), env=env, ) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxml2_dir), env=env) - subprocess.check_output( + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxml2_dir), env=env) + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxml2_dir), env=env ) self.info('Building libxslt') libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) - subprocess.check_output( + subprocess.check_call( [ './configure', prefix_arg, @@ -507,8 +507,8 @@ def prepare_static_build(self, build_platform): cwd=str(libxslt_dir), env=env, ) - subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxslt_dir), env=env) - subprocess.check_output( + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxslt_dir), env=env) + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxslt_dir), env=env ) @@ -516,7 +516,7 @@ def prepare_static_build(self, build_platform): ldflags.append('-lpthread') env['LDFLAGS'] = ' '.join(ldflags) xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) - subprocess.check_output( + subprocess.check_call( [ './configure', prefix_arg, @@ -537,13 +537,13 @@ def prepare_static_build(self, build_platform): cwd=str(xmlsec1_dir), env=env, ) - subprocess.check_output( + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1)] + ['-I{}'.format(str(self.prefix_dir / 'include')), '-I{}'.format(str(self.prefix_dir / 'include' / 'libxml'))], cwd=str(xmlsec1_dir), env=env, ) - subprocess.check_output( + subprocess.check_call( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(xmlsec1_dir), env=env ) From c63a4869f9facc60e810a01c3e1cd32d5d14a810 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:10:56 +0200 Subject: [PATCH 173/211] Download xmlsec1 releases from GitHub The server at www.aleksey.com (Cloudflare?) returns 403 Forbidden errors for some combination of: - missing User-Agent header - GitHub Actions CI runner IP address range - TLS settings (version, cipher suite) used by Python <= 3.9 --- setup.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index d5cf53aa..8383db4d 100644 --- a/setup.py +++ b/setup.py @@ -70,22 +70,18 @@ def latest_release_from_gnome_org_cache(url, lib_name): return '{}/{}'.format(url, latest_source) -def latest_release_from_github_api(repo): - api_url = 'https://api.github.com/repos/{}/releases'.format(repo) +def latest_release_json_from_github_api(repo): + api_url = 'https://api.github.com/repos/{}/releases/latest'.format(repo) # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited token = os.environ.get("GH_TOKEN") if token: log.info("Using GitHub token to avoid rate limiting") - api_releases = make_request(api_url, token, json_response=True) - releases = [r['tarball_url'] for r in api_releases if r['prerelease'] is False and r['draft'] is False] - if not releases: - raise DistutilsError('No release found for {}'.format(repo)) - return releases[0] + return make_request(api_url, token, json_response=True) def latest_openssl_release(): - return latest_release_from_github_api('openssl/openssl') + return latest_release_json_from_github_api('openssl/openssl')['tarball_url'] def latest_zlib_release(): @@ -105,7 +101,9 @@ def latest_libxslt_release(): def latest_xmlsec_release(): - return latest_release_from_html('https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz')) + assets = latest_release_json_from_github_api('lsh123/xmlsec')['assets'] + (tar_gz,) = [asset for asset in assets if asset['name'].endswith('.tar.gz')] + return tar_gz['browser_download_url'] class CrossCompileInfo: @@ -381,7 +379,7 @@ def prepare_static_build(self, build_platform): url = latest_xmlsec_release() self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {}'.format(url))) else: - url = 'https://www.aleksey.com/xmlsec/download/xmlsec1-{}.tar.gz'.format(self.xmlsec1_version) + url = 'https://github.com/lsh123/xmlsec/releases/download/{v}/xmlsec1-{v}.tar.gz'.format(v=self.xmlsec1_version) self.info( '{:10}: {}'.format( 'xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION={}, downloading from {}'.format(self.xmlsec1_version, url) From 7f1e1d9a48bcf2475f34ad67d25c5a7c0a0b7700 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:14:49 +0200 Subject: [PATCH 174/211] Fix empty argument to ./configure This solves the warning: configure: WARNING: you should use --build, --host, --target --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 8383db4d..c70381d9 100644 --- a/setup.py +++ b/setup.py @@ -445,9 +445,9 @@ def prepare_static_build(self, build_platform): subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) - host_arg = "" + host_arg = [] if cross_compiling: - host_arg = '--host={}'.format(cross_compiling.arch) + host_arg = ['--host={}'.format(cross_compiling.arch)] self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) @@ -457,7 +457,7 @@ def prepare_static_build(self, build_platform): prefix_arg, '--disable-dependency-tracking', '--disable-shared', - host_arg, + *host_arg, ], cwd=str(libiconv_dir), env=env, @@ -479,7 +479,7 @@ def prepare_static_build(self, build_platform): '--without-python', '--with-iconv={}'.format(self.prefix_dir), '--with-zlib={}'.format(self.prefix_dir), - host_arg, + *host_arg, ], cwd=str(libxml2_dir), env=env, @@ -500,7 +500,7 @@ def prepare_static_build(self, build_platform): '--without-python', '--without-crypto', '--with-libxml-prefix={}'.format(self.prefix_dir), - host_arg, + *host_arg, ], cwd=str(libxslt_dir), env=env, @@ -530,7 +530,7 @@ def prepare_static_build(self, build_platform): '--with-openssl={}'.format(self.prefix_dir), '--with-libxml={}'.format(self.prefix_dir), '--with-libxslt={}'.format(self.prefix_dir), - host_arg, + *host_arg, ], cwd=str(xmlsec1_dir), env=env, From 9c11b8fbf07131dd4709e0d09af89db64f53af9f Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:16:04 +0200 Subject: [PATCH 175/211] Update wheel availability based on lxml 6.0.0 --- pyproject.toml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8c03f910..540f91c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,12 +54,10 @@ skip = [ "*-musllinux_i686", # LXML doesn't publish wheels for these platforms, which makes it # difficult for us to build wheels, so we exclude them. - "cp36-manylinux_aarch64", - "cp37-manylinux_aarch64", - "cp36-musllinux_aarch64", - "cp37-musllinux_aarch64", - "cp36-macosx*", - "cp37-macosx*", + "cp36-*", + "cp37-*", + "cp38-manylinux_aarch64", + "cp38-musllinux_aarch64", "cp38-macosx*", ] test-command = "pytest -v --color=yes {package}/tests" From c3f3df2a301ef3e225448fd417aba9b463519cb4 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:16:36 +0200 Subject: [PATCH 176/211] Fix linuxbrew workflow --- .github/workflows/linuxbrew.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index d8996fd6..c3ede39b 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -6,6 +6,9 @@ jobs: strategy: matrix: python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + env: + # For some unknown reason, linuxbrew tries to use "gcc-11" by default, which doesn't exist. + CC: gcc steps: - uses: actions/checkout@v3 - name: Install brew From 749da1146a87bb1d574af2df0ab012ac2bd50476 Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:16:52 +0200 Subject: [PATCH 177/211] Fix macosx workflow --- .github/workflows/macosx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 1b82d22d..f639637e 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -26,6 +26,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" + export PYXMLSEC_LIBXML2_VERSION="$(pkg-config --modversion libxml-2.0)" python -m build rm -rf build/ - name: Set environment variables From 4f213157de1fa123b77754f9723709a1ff460c6f Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Mon, 7 Jul 2025 01:17:13 +0200 Subject: [PATCH 178/211] Fix manylinux workflow --- .github/workflows/manylinux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index a44776b3..db8ea913 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,11 +5,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] + python-abi: [cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313] image: - manylinux2014_x86_64 - manylinux_2_28_x86_64 - - musllinux_1_1_x86_64 + - musllinux_1_2_x86_64 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 From 8f047912b4a00f5d2f42dc55691d241fbfdd2d53 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 00:16:28 +0000 Subject: [PATCH 179/211] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index c70381d9..8d3d4b94 100644 --- a/setup.py +++ b/setup.py @@ -463,9 +463,7 @@ def prepare_static_build(self, build_platform): env=env, ) subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) - subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libiconv_dir), env=env - ) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libiconv_dir), env=env) self.info('Building LibXML2') libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) @@ -485,9 +483,7 @@ def prepare_static_build(self, build_platform): env=env, ) subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxml2_dir), env=env) - subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxml2_dir), env=env - ) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxml2_dir), env=env) self.info('Building libxslt') libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) @@ -506,9 +502,7 @@ def prepare_static_build(self, build_platform): env=env, ) subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxslt_dir), env=env) - subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxslt_dir), env=env - ) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxslt_dir), env=env) self.info('Building xmlsec1') ldflags.append('-lpthread') @@ -541,9 +535,7 @@ def prepare_static_build(self, build_platform): cwd=str(xmlsec1_dir), env=env, ) - subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(xmlsec1_dir), env=env - ) + subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(xmlsec1_dir), env=env) ext = self.ext_map['xmlsec'] ext.define_macros = [ From 3480dd30526faaf7764b5a674fa739dffa689a20 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:07:46 +0200 Subject: [PATCH 180/211] [pre-commit.ci] pre-commit autoupdate (#312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 24.3.0 → 25.1.0](https://github.com/psf/black/compare/24.3.0...25.1.0) - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/PyCQA/flake8: 7.0.0 → 7.3.0](https://github.com/PyCQA/flake8/compare/7.0.0...7.3.0) - [github.com/PyCQA/isort: 5.13.2 → 6.0.1](https://github.com/PyCQA/isort/compare/5.13.2...6.0.1) - [github.com/pre-commit/mirrors-mypy: v1.9.0 → v1.16.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.9.0...v1.16.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 820778ef..a02d6fac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 25.1.0 hooks: - id: black types: [] files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace @@ -25,17 +25,17 @@ repos: - id: pretty-format-json args: [--autofix] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.3.0 hooks: - id: flake8 exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 6.0.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.16.1 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From 8f3d9247259116ed856bcc0e840c1ee644ccefa7 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 10 Jul 2025 12:34:08 +0200 Subject: [PATCH 181/211] Fix windows wheels and add ARM support (#354) Co-authored-by: Tomi Belan --- .github/workflows/wheels.yml | 2 +- setup.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f9294fd3..66e6db8f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -87,7 +87,7 @@ jobs: && cibuildwheel --print-build-identifiers --platform macos \ | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ && cibuildwheel --print-build-identifiers --platform windows \ - | jq -nRc '{"only": inputs, "os": "windows-2019"}' + | jq -nRc '{"only": inputs, "os": "windows-2022"}' } | jq -sc ) echo "include=$MATRIX" diff --git a/setup.py b/setup.py index 8d3d4b94..7c4a2e63 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import json import multiprocessing import os +import platform import re import subprocess import sys @@ -211,19 +212,21 @@ def run(self): super(build_ext, self).run() def prepare_static_build_win(self): - release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2024.04.17/' - if sys.maxsize > 2147483647: # 2.0 GiB + release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' + if platform.machine() == 'ARM64': + suffix = 'win-arm64' + elif sys.maxsize > 2**32: # 2.0 GiB suffix = 'win64' else: suffix = 'win32' libs = [ - 'libxml2-2.11.7.{}.zip'.format(suffix), - 'libxslt-1.1.37.{}.zip'.format(suffix), - 'zlib-1.2.12.{}.zip'.format(suffix), - 'iconv-1.16-1.{}.zip'.format(suffix), - 'openssl-3.0.8.{}.zip'.format(suffix), - 'xmlsec-1.3.4.{}.zip'.format(suffix), + 'libxml2-2.11.9-3.{}.zip'.format(suffix), + 'libxslt-1.1.39.{}.zip'.format(suffix), + 'zlib-1.3.1.{}.zip'.format(suffix), + 'iconv-1.18-1.{}.zip'.format(suffix), + 'openssl-3.0.16.pl1.{}.zip'.format(suffix), + 'xmlsec-1.3.7.{}.zip'.format(suffix), ] for libfile in libs: From f7b8507b02fc17b7d96a1b4689ef11012cda8f00 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 10 Jul 2025 21:13:55 +0200 Subject: [PATCH 182/211] Add Windows ARM wheels support (#355) --- .github/workflows/wheels.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 66e6db8f..ae10888c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -87,7 +87,9 @@ jobs: && cibuildwheel --print-build-identifiers --platform macos \ | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ && cibuildwheel --print-build-identifiers --platform windows \ - | jq -nRc '{"only": inputs, "os": "windows-2022"}' + | jq -nRc '{"only": inputs, "os": "windows-2022"}' \ + && cibuildwheel --print-build-identifiers --platform windows --archs ARM64 \ + | jq -nRc '{"only": inputs, "os": "windows-11-arm"}' } | jq -sc ) echo "include=$MATRIX" From f09c3c86015d2b97f56d96900cfc02e70aaf5518 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 14 Jul 2025 07:55:53 +0200 Subject: [PATCH 183/211] Convert README format from reStructuredText to markdown (#357) Markdown is much easier to manage and more GitHub compatible. Also updated the badges links and removed deprecated ones. --- .appveyor.yml | 64 -------------- README.md | 198 +++++++++++++++++++++++++++++++++++++++++++ README.rst | 229 -------------------------------------------------- setup.cfg | 2 +- setup.py | 2 +- 5 files changed, 200 insertions(+), 295 deletions(-) delete mode 100644 .appveyor.yml create mode 100644 README.md delete mode 100644 README.rst diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index ce025819..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,64 +0,0 @@ -environment: - matrix: - - python: 35 - - python: 35-x64 - - python: 36 - - python: 36-x64 - - python: 37 - - python: 37-x64 - - python: 38 - - python: 38-x64 - - python: 39 - python_version: 3.9.13 - - python: 39-x64 - python_version: 3.9.13 - - python: 310 - python_version: 3.10.6 - - python: 310-x64 - python_version: 3.10.6 - - python: 311 - python_version: 3.11.2 - - python: 311-x64 - python_version: 3.11.2 - -install: - - ps: | - # from https://github.com/appveyor/build-images/blob/27bde614bc60d7ef7a8bc46182f4d7582fa11b56/scripts/Windows/install_python.ps1#L88-L108 - function InstallPythonEXE($targetPath, $version) { - $urlPlatform = "" - if ($targetPath -match '-x64$') { - $urlPlatform = "-amd64" - } - Write-Host "Installing Python $version$urlPlatform to $($targetPath)..." -ForegroundColor Cyan - $downloadUrl = "https://www.python.org/ftp/python/$version/python-$version$urlPlatform.exe" - Write-Host "Downloading $($downloadUrl)..." - $exePath = "$env:TEMP\python-$version.exe" - (New-Object Net.WebClient).DownloadFile($downloadUrl, $exePath) - Write-Host "Installing..." - cmd /c start /wait $exePath /quiet TargetDir="$targetPath" Shortcuts=0 Include_launcher=1 InstallLauncherAllUsers=1 Include_debug=1 - Remove-Item $exePath - Write-Host "Installed Python $version" -ForegroundColor Green - } - if ( -not ( Test-Path -Path C:\\Python$env:PYTHON -PathType Container ) ) { - InstallPythonEXE C:\\Python$env:PYTHON $env:PYTHON_VERSION - } - - SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH% - - python -m pip install -U pip wheel setuptools - -build: off -build_script: - - python setup.py bdist_wheel - -test: off -test_script: - - pip install -r requirements-test.txt - - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist - - pytest -v --color=yes --junitxml=unittests.xml - - ps: Get-ChildItem dist\*.whl | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } - -on_finish: - - ps: | - # archive test results at AppVeyor - $wc = New-Object 'System.Net.WebClient' - $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\unittests.xml)) - $LastExitCode = 0 diff --git a/README.md b/README.md new file mode 100644 index 00000000..6abd6a5a --- /dev/null +++ b/README.md @@ -0,0 +1,198 @@ +# python-xmlsec + +[![image](https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/xmlsec) +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/xmlsec/python-xmlsec/master.svg)](https://results.pre-commit.ci/latest/github/xmlsec/python-xmlsec/master) +[![image](https://github.com/xmlsec/python-xmlsec/actions/workflows/manylinux.yml/badge.svg)](https://github.com/xmlsec/python-xmlsec/actions/workflows/manylinux.yml) +[![image](https://github.com/xmlsec/python-xmlsec/actions/workflows/macosx.yml/badge.svg)](https://github.com/xmlsec/python-xmlsec/actions/workflows/macosx.yml) +[![image](https://github.com/xmlsec/python-xmlsec/actions/workflows/linuxbrew.yml/badge.svg)](https://github.com/xmlsec/python-xmlsec/actions/workflows/linuxbrew.yml) +[![image](https://codecov.io/gh/xmlsec/python-xmlsec/branch/master/graph/badge.svg)](https://codecov.io/gh/xmlsec/python-xmlsec) +[![Documentation Status](https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs)](https://xmlsec.readthedocs.io/en/latest/?badge=latest) + +Python bindings for the [XML Security +Library](https://www.aleksey.com/xmlsec/). + +## Documentation + +Documentation for `xmlsec` can be found at +[xmlsec.readthedocs.io](https://xmlsec.readthedocs.io/). + +## Usage + +Check the +[examples](https://xmlsec.readthedocs.io/en/latest/examples.html) +section in the documentation to see various examples of signing and +verifying using the library. + +## Requirements + +- `libxml2 >= 2.9.1` +- `libxmlsec1 >= 1.2.33` + +## Install + +`xmlsec` is available on PyPI: + +``` bash +pip install xmlsec +``` + +Depending on your OS, you may need to install the required native +libraries first: + +### Linux (Debian) + +``` bash +apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl +``` + +Note: There is no required version of LibXML2 for Ubuntu Precise, so you +need to download and install it manually. + +``` bash +wget http://xmlsoft.org/sources/libxml2-2.9.1.tar.gz +tar -xvf libxml2-2.9.1.tar.gz +cd libxml2-2.9.1 +./configure && make && make install +``` + +### Linux (CentOS) + +``` bash +yum install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel +``` + +### Linux (Fedora) + +``` bash +dnf install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel +``` + +### Mac + +``` bash +brew install libxml2 libxmlsec1 pkg-config +``` + +or + +``` bash +port install libxml2 xmlsec pkgconfig +``` + +### Alpine + +``` bash +apk add build-base libressl libffi-dev libressl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec +``` + +## Troubleshooting + +### Mac + +If you get any fatal errors about missing `.h` files, update your +`C_INCLUDE_PATH` environment variable to include the appropriate files +from the `libxml2` and `libxmlsec1` libraries. + +### Windows + +Starting with 1.3.7, prebuilt wheels are available for Windows, so +running `pip install xmlsec` should suffice. If you want to build from +source: + +1. Configure build environment, see + [wiki.python.org](https://wiki.python.org/moin/WindowsCompilers) for + more details. + +2. Install from source dist: + + ``` bash + pip install xmlsec --no-binary=xmlsec + ``` + +## Building from source + +1. Clone the `xmlsec` source code repository to your local computer. + + ``` bash + git clone https://github.com/xmlsec/python-xmlsec.git + ``` + +2. Change into the `python-xmlsec` root directory. + + ``` bash + cd /path/to/xmlsec + ``` + +3. Install the project and all its dependencies using `pip`. + + ``` bash + pip install . + ``` + +## Contributing + +### Setting up your environment + +1. Follow steps 1 and 2 of the [manual installation + instructions](#building-from-source). + +2. Initialize a virtual environment to develop in. This is done so as + to ensure every contributor is working with close-to-identical + versions of packages. + + ``` bash + mkvirtualenv xmlsec + ``` + + The `mkvirtualenv` command is available from `virtualenvwrapper` + package which can be installed by following + [link](http://virtualenvwrapper.readthedocs.org/en/latest/install.html#basic-installation). + +3. Activate the created virtual environment: + + ``` bash + workon xmlsec + ``` + +4. Install `xmlsec` in development mode with testing enabled. This will + download all dependencies required for running the unit tests. + + ``` bash + pip install -r requirements-test.txt + pip install -e "." + ``` + +### Running the test suite + +1. [Set up your environment](#setting-up-your-environment). + +2. Run the unit tests. + + ``` bash + pytest tests + ``` + +3. Tests configuration + + Env variable `PYXMLSEC_TEST_ITERATIONS` specifies number of test + iterations to detect memory leaks. + +### Reporting an issue + +Please attach the output of following information: + +- version of `xmlsec` +- version of `libxmlsec1` +- version of `libxml2` +- output from the command + + ``` bash + pkg-config --cflags xmlsec1 + ``` + +## License + +Unless otherwise noted, all files contained within this project are +licensed under the MIT open source license. See the included `LICENSE` +file or visit [opensource.org](http://opensource.org/licenses/MIT) for +more information. diff --git a/README.rst b/README.rst deleted file mode 100644 index bd3a1be9..00000000 --- a/README.rst +++ /dev/null @@ -1,229 +0,0 @@ -python-xmlsec -============= - -.. image:: https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white - :target: https://pypi.python.org/pypi/xmlsec -.. image:: https://results.pre-commit.ci/badge/github/xmlsec/python-xmlsec/master.svg - :target: https://results.pre-commit.ci/latest/github/xmlsec/python-xmlsec/master - :alt: pre-commit.ci status -.. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor - :target: https://ci.appveyor.com/project/hoefling/xmlsec -.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml -.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/macosx.yml/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions/workflows/macosx.yml -.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml -.. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml/badge.svg - :target: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml -.. image:: https://codecov.io/gh/xmlsec/python-xmlsec/branch/master/graph/badge.svg - :target: https://codecov.io/gh/xmlsec/python-xmlsec -.. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs - :target: https://xmlsec.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -Python bindings for the `XML Security Library `_. - -Documentation -************* - -A documentation for ``xmlsec`` can be found at `xmlsec.readthedocs.io `_. - -Usage -***** - -Check the `examples `_ section in the documentation to see various examples of signing and verifying using the library. - -Requirements -************ -- ``libxml2 >= 2.9.1`` -- ``libxmlsec1 >= 1.2.33`` - -Install -******* - -``xmlsec`` is available on PyPI: - -.. code-block:: bash - - pip install xmlsec - -Depending on your OS, you may need to install the required native -libraries first: - -Linux (Debian) -^^^^^^^^^^^^^^ - -.. code-block:: bash - - apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl - - -Note: There is no required version of LibXML2 for Ubuntu Precise, -so you need to download and install it manually. - -.. code-block:: bash - - wget http://xmlsoft.org/sources/libxml2-2.9.1.tar.gz - tar -xvf libxml2-2.9.1.tar.gz - cd libxml2-2.9.1 - ./configure && make && make install - - -Linux (CentOS) -^^^^^^^^^^^^^^ - -.. code-block:: bash - - yum install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel - - -Linux (Fedora) -^^^^^^^^^^^^^^ - -.. code-block:: bash - - dnf install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel - - -Mac -^^^ - -.. code-block:: bash - - brew install libxml2 libxmlsec1 pkg-config - -or - -.. code-block:: bash - - port install libxml2 xmlsec pkgconfig - - -Alpine -^^^^^^ - -.. code-block:: bash - - apk add build-base libressl libffi-dev libressl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec - - -Troubleshooting -*************** - -Mac -^^^ - -If you get any fatal errors about missing ``.h`` files, update your -``C_INCLUDE_PATH`` environment variable to include the appropriate -files from the ``libxml2`` and ``libxmlsec1`` libraries. - - -Windows -^^^^^^^ - -Starting with 1.3.7, prebuilt wheels are available for Windows, -so running ``pip install xmlsec`` should suffice. If you want -to build from source: - -#. Configure build environment, see `wiki.python.org `_ for more details. - -#. Install from source dist: - - .. code-block:: bash - - pip install xmlsec --no-binary=xmlsec - - -Building from source -******************** - -#. Clone the ``xmlsec`` source code repository to your local computer. - - .. code-block:: bash - - git clone https://github.com/xmlsec/python-xmlsec.git - -#. Change into the ``python-xmlsec`` root directory. - - .. code-block:: bash - - cd /path/to/xmlsec - - -#. Install the project and all its dependencies using ``pip``. - - .. code-block:: bash - - pip install . - - -Contributing -************ - -Setting up your environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -#. Follow steps 1 and 2 of the `manual installation instructions <#building-from-source>`_. - - -#. Initialize a virtual environment to develop in. - This is done so as to ensure every contributor is working with - close-to-identicial versions of packages. - - .. code-block:: bash - - mkvirtualenv xmlsec - - The ``mkvirtualenv`` command is available from ``virtualenvwrapper`` package which can be installed by following `link `_. - -#. Activate the created virtual environment: - - .. code-block:: bash - - workon xmlsec - -#. Install ``xmlsec`` in development mode with testing enabled. - This will download all dependencies required for running the unit tests. - - .. code-block:: bash - - pip install -r requirements-test.txt - pip install -e "." - - -Running the test suite -^^^^^^^^^^^^^^^^^^^^^^ - -#. `Set up your environment <#setting-up-your-environment>`_. - -#. Run the unit tests. - - .. code-block:: bash - - pytest tests - -#. Tests configuration - - Env variable ``PYXMLSEC_TEST_ITERATIONS`` specifies number of - test iterations to detect memory leaks. - -Reporting an issue -^^^^^^^^^^^^^^^^^^ - -Please attach the output of following information: - -* version of ``xmlsec`` -* version of ``libxmlsec1`` -* version of ``libxml2`` -* output from the command - - .. code-block:: bash - - pkg-config --cflags xmlsec1 - -License -******* - -Unless otherwise noted, all files contained within this project are licensed under the MIT opensource license. -See the included ``LICENSE`` file or visit `opensource.org `_ for more information. diff --git a/setup.cfg b/setup.cfg index c090b4e8..7fee05f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description_file = README.rst +description_file = README.md [bdist_rpm] release = 1 diff --git a/setup.py b/setup.py index 7c4a2e63..447cebc4 100644 --- a/setup.py +++ b/setup.py @@ -584,7 +584,7 @@ def prepare_static_build(self, build_platform): setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] -with io.open('README.rst', encoding='utf-8') as f: +with io.open('README.md', encoding='utf-8') as f: long_desc = f.read() From 63ab5969bd65be570acb4bca070c46ace4c78b40 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 14 Jul 2025 10:27:30 +0200 Subject: [PATCH 184/211] Drop legacy platform support and improve CI efficiency (#358) Cancelled in-progress GitHub Actions using concurrency groups to reduce CI resource usage. Fully removed support for Python 3.8 and lower. Discontinued building wheels for Linux i686 and Windows 32-bit due to low usage based on PyPI download statistics. --- .github/workflows/linuxbrew.yml | 3 ++ .github/workflows/macosx.yml | 5 ++- .github/workflows/manylinux.yml | 5 ++- .github/workflows/sdist.yml | 3 ++ .github/workflows/wheels.yml | 4 ++ .travis.yml | 79 +++++++++++++++++---------------- pyproject.toml | 21 +++++---- 7 files changed, 68 insertions(+), 52 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index c3ede39b..404321ea 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -1,5 +1,8 @@ name: linuxbrew on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: linuxbrew: runs-on: ubuntu-latest diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index f639637e..7961b5b9 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -1,5 +1,8 @@ -name: MacOS +name: macOS on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: macosx: runs-on: macos-latest diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index db8ea913..9d578f05 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -1,11 +1,14 @@ name: manylinux on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: manylinux: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313] + python-abi: [cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313] image: - manylinux2014_x86_64 - manylinux_2_28_x86_64 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 04f22c14..3f3eacf0 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -1,5 +1,8 @@ name: sdist on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: sdist: # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ae10888c..6b74c1b6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -15,6 +15,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: {} jobs: diff --git a/.travis.yml b/.travis.yml index 9e6ca540..9574a1c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,51 +1,52 @@ -dist: trusty -sudo: false +dist: focal language: python +travis: + auto_cancel: + push: true + pull_request: true + notifications: email: false -matrix: - include: - - python: 3.5 - - python: 3.6 - - python: 3.7 - dist: xenial - sudo: required - - python: 3.8 - dist: xenial - sudo: required - - python: 3.9 - dist: xenial - sudo: required - - python: 3.11 - dist: xenial - sudo: required + +python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + env: global: - - CFLAGS=-coverage - - LDFLAGS=-coverage -lgcov - - PYXMLSEC_TEST_ITERATIONS=50 + - CFLAGS=-coverage + - LDFLAGS=-coverage -lgcov + - PYXMLSEC_TEST_ITERATIONS=50 addons: apt: packages: - - libssl-dev - - libxmlsec1 - - libxmlsec1-dev - - libxmlsec1-openssl - - libxslt1-dev - - pkg-config - - lcov + - libssl-dev + - libxmlsec1 + - libxmlsec1-dev + - libxmlsec1-openssl + - libxslt1-dev + - pkg-config + - lcov + install: -- travis_retry pip install --upgrade pip setuptools wheel -- travis_retry pip install coverage -r requirements-test.txt --upgrade --force-reinstall -- python setup.py bdist_wheel -- pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ -script: coverage run -m pytest -v tests --color=yes + - travis_retry pip install --upgrade pip setuptools wheel + - travis_retry pip install coverage -r requirements-test.txt --upgrade --force-reinstall + - python setup.py bdist_wheel + - pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ + +script: + - coverage run -m pytest -v tests --color=yes + after_success: -- lcov --capture --no-external --directory . --output-file coverage.info -- lcov --list coverage.info -- bash <(curl -s https://codecov.io/bash) -f coverage.info + - lcov --capture --no-external --directory . --output-file coverage.info + - lcov --list coverage.info + - bash <(curl -s https://codecov.io/bash) -f coverage.info + before_deploy: -- travis_retry pip install Sphinx -r doc/source/requirements.txt -- git apply --verbose --no-index --unsafe-paths --directory=$(python -c "import site; print(site.getsitepackages()[0])") doc/source/sphinx-pr-6916.diff -- sphinx-build -EWanb html doc/source build/sphinx + - travis_retry pip install Sphinx -r doc/source/requirements.txt + - git apply --verbose --no-index --unsafe-paths --directory=$(python -c "import site; print(site.getsitepackages()[0])") doc/source/sphinx-pr-6916.diff + - sphinx-build -EWanb html doc/source build/sphinx diff --git a/pyproject.toml b/pyproject.toml index 540f91c1..a0380dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,18 +47,17 @@ known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] [tool.cibuildwheel] +build = [ + "cp39-*", + "cp310-*", + "cp311-*", + "cp312-*", + "cp313-*" +] build-verbosity = 1 build-frontend = "build" skip = [ - "pp*", - "*-musllinux_i686", - # LXML doesn't publish wheels for these platforms, which makes it - # difficult for us to build wheels, so we exclude them. - "cp36-*", - "cp37-*", - "cp38-manylinux_aarch64", - "cp38-musllinux_aarch64", - "cp38-macosx*", + "pp*", # Skips PyPy builds (pp38-*, pp39-*, etc.) ] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" @@ -68,7 +67,7 @@ test-skip = "*-macosx_arm64" PYXMLSEC_STATIC_DEPS = "true" [tool.cibuildwheel.linux] -archs = ["x86_64", "aarch64", "i686"] +archs = ["x86_64", "aarch64"] environment-pass = [ "PYXMLSEC_LIBXML2_VERSION", "PYXMLSEC_LIBXSLT_VERSION", @@ -81,7 +80,7 @@ archs = ["x86_64", "arm64"] before-all = "brew install perl" [tool.cibuildwheel.windows] -archs = ["AMD64", "x86"] +archs = ["AMD64"] [[tool.cibuildwheel.overrides]] select = "*-manylinux*" From f43021900f9de3b9fb60f933720d38c06b885d84 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 14 Jul 2025 16:29:06 +0200 Subject: [PATCH 185/211] Migrate linting to Ruff and update CI settings (#359) Replace flake8, black, and isort with Ruff as the unified linter and formatter. - Applied auto-fixes from Ruff - Removed legacy configuration for flake8, black, and isort - Updated CI settings to prevent job cancelation on master --- .github/workflows/linuxbrew.yml | 2 +- .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 2 +- .github/workflows/sdist.yml | 2 +- .github/workflows/wheels.yml | 2 +- .pre-commit-config.yaml | 84 +++++++-------- doc/source/conf.py | 25 +++-- doc/source/examples/decrypt.py | 4 +- doc/source/examples/encrypt.py | 24 ++--- doc/source/examples/verify.py | 2 +- pyproject.toml | 88 +++++++++++----- requirements-test.txt | 6 +- requirements.txt | 2 +- setup.cfg | 19 ++-- setup.py | 178 ++++++++++++++------------------ src/xmlsec/constants.pyi | 7 +- tests/base.py | 32 +++--- tests/conftest.py | 3 +- tests/softhsm_setup.py | 26 ++--- tests/test_constants.py | 8 +- tests/test_doc_examples.py | 14 +-- tests/test_ds.py | 110 ++++++++++---------- tests/test_enc.py | 50 ++++----- tests/test_keys.py | 120 +++++++++++---------- tests/test_main.py | 12 +-- tests/test_pkcs11.py | 8 +- tests/test_templates.py | 88 ++++++++-------- tests/test_tree.py | 10 +- tests/test_type_stubs.py | 8 +- tests/test_xmlsec.py | 3 +- 30 files changed, 460 insertions(+), 481 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 404321ea..8a231047 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -2,7 +2,7 @@ name: linuxbrew on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'master' }} jobs: linuxbrew: runs-on: ubuntu-latest diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 7961b5b9..44d214bc 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -2,7 +2,7 @@ name: macOS on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'master' }} jobs: macosx: runs-on: macos-latest diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 9d578f05..357e93eb 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -2,7 +2,7 @@ name: manylinux on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'master' }} jobs: manylinux: runs-on: ubuntu-latest diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 3f3eacf0..3bdc9764 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -2,7 +2,7 @@ name: sdist on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'master' }} jobs: sdist: # Avoid Ubuntu 24.04 in sdist workflows, because it contains libxmlsec1-dev diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 6b74c1b6..62d48b4b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -17,7 +17,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_name != 'master' }} permissions: {} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a02d6fac..48abcc6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,48 +1,42 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/psf/black - rev: 25.1.0 - hooks: - - id: black - types: [] - files: ^.*.pyi?$ - exclude: ^doc/ -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - - id: no-commit-to-branch - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files - - id: check-ast - - id: check-merge-conflict - - id: check-json - - id: detect-private-key - exclude: ^.*/rsakey.pem$ - - id: mixed-line-ending - - id: pretty-format-json - args: [--autofix] -- repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 - hooks: - - id: flake8 - exclude: ^setup.py$ - additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] -- repo: https://github.com/PyCQA/isort - rev: 6.0.1 - hooks: - - id: isort -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.1 - hooks: - - id: mypy - exclude: (setup.py|tests/.*.py|doc/.*) - types: [] - files: ^.*.pyi?$ - additional_dependencies: [lxml-stubs,types-docutils] -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.10.0 - hooks: - - id: rst-backticks +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.3 + hooks: + - id: ruff + args: ["--fix"] + types: [python] + - id: ruff-format + types: [python] + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: no-commit-to-branch + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-ast + - id: check-merge-conflict + - id: check-json + - id: detect-private-key + exclude: ^.*/rsakey.pem$ + - id: mixed-line-ending + - id: pretty-format-json + args: [--autofix] + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.16.1 + hooks: + - id: mypy + exclude: (setup.py|tests/.*.py|doc/.*) + types: [] + files: ^.*.pyi?$ + additional_dependencies: [lxml-stubs, types-docutils] + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks diff --git a/doc/source/conf.py b/doc/source/conf.py index 35329050..900b79da 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,12 +19,12 @@ source_suffix = '.rst' master_doc = 'index' -project = u'python-xmlsec' -copyright = u'2020, Oleg Hoefling ' # noqa: A001 -author = u'Bulat Gaifullin ' +project = 'python-xmlsec' +copyright = '2020, Oleg Hoefling ' +author = 'Bulat Gaifullin ' release = importlib.metadata.version('xmlsec') parsed: Version = parse(release) -version = '{}.{}'.format(parsed.major, parsed.minor) +version = f'{parsed.major}.{parsed.minor}' exclude_patterns: list[str] = [] pygments_style = 'sphinx' @@ -39,19 +39,19 @@ ( master_doc, 'python-xmlsec.tex', - u'python-xmlsec Documentation', - u'Bulat Gaifullin \\textless{}gaifullinbf@gmail.com\\textgreater{}', + 'python-xmlsec Documentation', + 'Bulat Gaifullin \\textless{}gaifullinbf@gmail.com\\textgreater{}', 'manual', ) ] -man_pages = [(master_doc, 'python-xmlsec', u'python-xmlsec Documentation', [author], 1)] +man_pages = [(master_doc, 'python-xmlsec', 'python-xmlsec Documentation', [author], 1)] texinfo_documents = [ ( master_doc, 'python-xmlsec', - u'python-xmlsec Documentation', + 'python-xmlsec Documentation', author, 'python-xmlsec', 'One line description of project.', @@ -63,10 +63,10 @@ autodoc_docstring_signature = True -rst_prolog = ''' +rst_prolog = """ .. role:: xml(code) :language: xml -''' +""" # LXML crossref'ing stuff: # LXML doesn't have an intersphinx docs, @@ -75,8 +75,7 @@ def lxml_element_doc_reference(app: Sphinx, env: BuildEnvironment, node: pending_xref, contnode: Text) -> reference: - """ - Handle a missing reference only if it is a ``lxml.etree._Element`` ref. + """Handle a missing reference only if it is a ``lxml.etree._Element`` ref. We handle only :class:`lxml.etree._Element` and :class:`~lxml.etree._Element` nodes. """ @@ -85,7 +84,7 @@ def lxml_element_doc_reference(app: Sphinx, env: BuildEnvironment, node: pending and node.get('reftarget', None) == 'lxml.etree._Element' and contnode.astext() in ('lxml.etree._Element', '_Element') ): - reftitle = '(in lxml v{})'.format(lxml.__version__) # type: ignore[attr-defined] + reftitle = f'(in lxml v{lxml.__version__})' # type: ignore[attr-defined] newnode = reference('', '', internal=False, refuri=lxml_element_cls_doc_uri, reftitle=reftitle) newnode.append(contnode) return newnode diff --git a/doc/source/examples/decrypt.py b/doc/source/examples/decrypt.py index e107756f..cb474d22 100644 --- a/doc/source/examples/decrypt.py +++ b/doc/source/examples/decrypt.py @@ -6,7 +6,7 @@ key = xmlsec.Key.from_file('rsakey.pem', xmlsec.constants.KeyDataFormatPem) manager.add_key(key) enc_ctx = xmlsec.EncryptionContext(manager) -root = etree.parse("enc1-res.xml").getroot() -enc_data = xmlsec.tree.find_child(root, "EncryptedData", xmlsec.constants.EncNs) +root = etree.parse('enc1-res.xml').getroot() +enc_data = xmlsec.tree.find_child(root, 'EncryptedData', xmlsec.constants.EncNs) decrypted = enc_ctx.decrypt(enc_data) print(etree.tostring(decrypted)) diff --git a/doc/source/examples/encrypt.py b/doc/source/examples/encrypt.py index 98f63b6f..2a92264e 100644 --- a/doc/source/examples/encrypt.py +++ b/doc/source/examples/encrypt.py @@ -9,11 +9,11 @@ template, xmlsec.constants.TransformAes128Cbc, type=xmlsec.constants.TypeEncElement, - ns="xenc", + ns='xenc', ) xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) -key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") +key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') enc_key = xmlsec.template.add_encrypted_key(key_info, xmlsec.constants.TransformRsaOaep) xmlsec.template.encrypted_data_ensure_cipher_value(enc_key) data = template.find('./Data') @@ -24,20 +24,10 @@ manager.add_key(key) enc_ctx = xmlsec.EncryptionContext(manager) -enc_ctx.key = xmlsec.Key.generate( - xmlsec.constants.KeyDataAes, 128, xmlsec.constants.KeyDataTypeSession -) +enc_ctx.key = xmlsec.Key.generate(xmlsec.constants.KeyDataAes, 128, xmlsec.constants.KeyDataTypeSession) enc_data = enc_ctx.encrypt_xml(enc_data, data) -enc_method = xmlsec.tree.find_child( - enc_data, xmlsec.constants.NodeEncryptionMethod, xmlsec.constants.EncNs -) -key_info = xmlsec.tree.find_child( - enc_data, xmlsec.constants.NodeKeyInfo, xmlsec.constants.DSigNs -) -enc_method = xmlsec.tree.find_node( - key_info, xmlsec.constants.NodeEncryptionMethod, xmlsec.constants.EncNs -) -cipher_value = xmlsec.tree.find_node( - key_info, xmlsec.constants.NodeCipherValue, xmlsec.constants.EncNs -) +enc_method = xmlsec.tree.find_child(enc_data, xmlsec.constants.NodeEncryptionMethod, xmlsec.constants.EncNs) +key_info = xmlsec.tree.find_child(enc_data, xmlsec.constants.NodeKeyInfo, xmlsec.constants.DSigNs) +enc_method = xmlsec.tree.find_node(key_info, xmlsec.constants.NodeEncryptionMethod, xmlsec.constants.EncNs) +cipher_value = xmlsec.tree.find_node(key_info, xmlsec.constants.NodeCipherValue, xmlsec.constants.EncNs) print(etree.tostring(cipher_value)) diff --git a/doc/source/examples/verify.py b/doc/source/examples/verify.py index 808a53c2..c3240c99 100644 --- a/doc/source/examples/verify.py +++ b/doc/source/examples/verify.py @@ -5,7 +5,7 @@ with open('sign1-res.xml') as fp: template = etree.parse(fp).getroot() -xmlsec.tree.add_ids(template, ["ID"]) +xmlsec.tree.add_ids(template, ['ID']) signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) # Create a digital signature context (no key manager is needed). ctx = xmlsec.SignatureContext() diff --git a/pyproject.toml b/pyproject.toml index a0380dcf..bb0c459e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4", "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] + [tool.mypy] files = ['src'] ignore_missing_imports = false @@ -19,32 +22,71 @@ warn_no_return = true no_implicit_reexport = true show_error_codes = true -[tool.black] -line_length = 130 -skip-string-normalization = true -target_version = ['py39'] -include = '\.pyi?$' -exclude = ''' +[tool.ruff] +# Maximum line length, same as your original Black + Flake8 config +line-length = 130 + +# Target Python version (used for autofixes and style rules) +target-version = "py39" -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.mypy_cache - | \.tox - | build - | dist - )/ -) -''' +# Directories and files to exclude from linting and formatting +exclude = [ + ".venv*", # virtual environments + ".git", # git directory + "build", # build output + "dist", # distribution packages + "libs", # vendor libraries + ".eggs", # setuptools egg folders + ".direnv*", # direnv environments + "*_pb2.pyi" # protobuf-generated type stubs +] -[tool.isort] -profile = 'black' -known_first_party = ['xmlsec'] -known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] +[tool.ruff.lint] +# Enable rule categories: +# E = pycodestyle (style issues, like indentation, whitespace, etc.) +# F = pyflakes (unused imports, undefined names) +# I = isort (import sorting) +# B = flake8-bugbear (common bugs & anti-patterns) +# UP = pyupgrade (auto-upgrade syntax for newer Python) +# SIM = flake8-simplify (simplifiable code patterns) +# RUF = Ruff-native rules (extra, performance-optimized checks) +select = ["E", "F", "I", "B", "UP", "SIM", "RUF"] +# TODO: Add more rule categories as needed, e.g.: +# D = pydocstyle (docstring format/style issues) -[build-system] -requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] +[tool.ruff.lint.per-file-ignores] +"*.pyi" = [ + # Ignore formatting and import errors in stub files + "E301", # expected 1 blank line, found 0 + "E302", # expected 2 blank lines, found 1 + "E305", # expected 2 blank lines after class or function + "E501", # line too long + "E701", # multiple statements on one line + "F401", # unused import + "F811", # redefinition of unused name + "F822" # undefined name in `__all__` +] +"doc/source/conf.py" = [ + "D1" # missing docstring in public module/class/function +] +"doc/source/examples/*.py" = [ + "D1", # allow missing docstrings in examples + "E501" # allow long lines in code examples +] +"tests/*.py" = [ + "D1" # allow missing docstrings in test files +] + +[tool.ruff.format] +# Always use single quotes (e.g., 'text' instead of "text") +quote-style = "single" + +# Format code with or without trailing commas +# true = prefer trailing commas where valid +skip-magic-trailing-comma = false + +# Enforce Unix-style line endings (LF) +line-ending = "lf" [tool.cibuildwheel] build = [ diff --git a/requirements-test.txt b/requirements-test.txt index eb543402..52a31f00 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,5 @@ -r requirements.txt -pytest>=4.6.9 -lxml-stubs + +pytest==8.4.1 +lxml-stubs==0.5.1 +ruff[format]==0.12.3 diff --git a/requirements.txt b/requirements.txt index 014bfe10..7aa6718b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -lxml >= 3.8.0, !=4.7.0 +lxml==6.0.0 diff --git a/setup.cfg b/setup.cfg index 7fee05f9..8762c654 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,14 +12,11 @@ source-dir = doc/source build-dir = doc/build all_files = 1 -[upload_docs] -upload_dir = doc/build/html - -[flake8] -per-file-ignores = - *.pyi: E301, E302, E305, E501, E701, F401, F822 - doc/source/conf.py: D1 - doc/source/examples/*.py: D1, E501 - tests/*.py: D1 -exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* -max-line-length = 130 +# [flake8] +# per-file-ignores = +# *.pyi: E301, E302, E305, E501, E701, F401, F822 +# doc/source/conf.py: D1 +# doc/source/examples/*.py: D1, E501 +# tests/*.py: D1 +# exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* +# max-line-length = 130 diff --git a/setup.py b/setup.py index 447cebc4..94b49aa8 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import contextlib import html.parser -import io import json import multiprocessing import os @@ -36,7 +35,7 @@ def handle_starttag(self, tag, attrs): def make_request(url, github_token=None, json_response=False): headers = {'User-Agent': 'https://github.com/xmlsec/python-xmlsec'} if github_token: - headers['authorization'] = "Bearer " + github_token + headers['authorization'] = 'Bearer ' + github_token request = Request(url, headers=headers) with contextlib.closing(urlopen(request)) as r: charset = r.headers.get_content_charset() or 'utf-8' @@ -60,24 +59,24 @@ def comp(text): return Version('0.0') latest = max(hrefs, key=comp) - return '{}/{}'.format(url, latest) + return f'{url}/{latest}' def latest_release_from_gnome_org_cache(url, lib_name): - cache_url = '{}/cache.json'.format(url) + cache_url = f'{url}/cache.json' cache = make_request(cache_url, json_response=True) latest_version = cache[2][lib_name][-1] latest_source = cache[1][lib_name][latest_version]['tar.xz'] - return '{}/{}'.format(url, latest_source) + return f'{url}/{latest_source}' def latest_release_json_from_github_api(repo): - api_url = 'https://api.github.com/repos/{}/releases/latest'.format(repo) + api_url = f'https://api.github.com/repos/{repo}/releases/latest' # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited - token = os.environ.get("GH_TOKEN") + token = os.environ.get('GH_TOKEN') if token: - log.info("Using GitHub token to avoid rate limiting") + log.info('Using GitHub token to avoid rate limiting') return make_request(api_url, token, json_response=True) @@ -115,7 +114,7 @@ def __init__(self, host, arch, compiler): @property def triplet(self): - return "{}-{}-{}".format(self.host, self.arch, self.compiler) + return f'{self.host}-{self.arch}-{self.compiler}' class build_ext(build_ext_orig): @@ -129,7 +128,7 @@ def run(self): self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) if self.static or sys.platform == 'win32': - self.info('starting static build on {}'.format(sys.platform)) + self.info(f'starting static build on {sys.platform}') buildroot = Path('build', 'tmp') self.prefix_dir = buildroot / 'prefix' @@ -145,19 +144,17 @@ def run(self): if sys.platform == 'win32': self.prepare_static_build_win() - elif 'linux' in sys.platform: - self.prepare_static_build(sys.platform) - elif 'darwin' in sys.platform: + elif 'linux' in sys.platform or 'darwin' in sys.platform: self.prepare_static_build(sys.platform) else: import pkgconfig try: config = pkgconfig.parse('xmlsec1') - except EnvironmentError: - raise DistutilsError('Unable to invoke pkg-config.') - except pkgconfig.PackageNotFoundError: - raise DistutilsError('xmlsec1 is not installed or not in path.') + except OSError as e: + raise DistutilsError('Unable to invoke pkg-config.') from e + except pkgconfig.PackageNotFoundError as e: + raise DistutilsError('xmlsec1 is not installed or not in path.') from e if config is None or not config.get('libraries'): raise DistutilsError('Bad or incomplete result returned from pkg-config.') @@ -178,7 +175,7 @@ def run(self): for key, value in ext.define_macros: if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): ext.define_macros.remove((key, value)) - ext.define_macros.append((key, '"{0}"'.format(value))) + ext.define_macros.append((key, f'"{value}"')) break if sys.platform == 'win32': @@ -209,7 +206,7 @@ def run(self): else: ext.extra_compile_args.append('-Os') - super(build_ext, self).run() + super().run() def prepare_static_build_win(self): release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' @@ -221,21 +218,21 @@ def prepare_static_build_win(self): suffix = 'win32' libs = [ - 'libxml2-2.11.9-3.{}.zip'.format(suffix), - 'libxslt-1.1.39.{}.zip'.format(suffix), - 'zlib-1.3.1.{}.zip'.format(suffix), - 'iconv-1.18-1.{}.zip'.format(suffix), - 'openssl-3.0.16.pl1.{}.zip'.format(suffix), - 'xmlsec-1.3.7.{}.zip'.format(suffix), + f'libxml2-2.11.9-3.{suffix}.zip', + f'libxslt-1.1.39.{suffix}.zip', + f'zlib-1.3.1.{suffix}.zip', + f'iconv-1.18-1.{suffix}.zip', + f'openssl-3.0.16.pl1.{suffix}.zip', + f'xmlsec-1.3.7.{suffix}.zip', ] for libfile in libs: url = urljoin(release_url, libfile) destfile = self.libs_dir / libfile if destfile.is_file(): - self.info('Using local copy of "{}"'.format(url)) + self.info(f'Using local copy of "{url}"') else: - self.info('Retrieving "{}" to "{}"'.format(url, destfile)) + self.info(f'Retrieving "{url}" to "{destfile}"') urlcleanup() # work around FTP bug 27973 in Py2.7.12+ urlretrieve(url, str(destfile)) @@ -296,9 +293,9 @@ def prepare_static_build(self, build_platform): openssl_tar = self.libs_dir / 'openssl.tar.gz' if self.openssl_version is None: url = latest_openssl_release() - self.info('{:10}: {}'.format('OpenSSL', 'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('OpenSSL', f'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {url}')) else: - url = 'https://api.github.com/repos/openssl/openssl/tarball/openssl-{}'.format(self.openssl_version) + url = f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{self.openssl_version}' self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) urlretrieve(url, str(openssl_tar)) @@ -309,12 +306,10 @@ def prepare_static_build(self, build_platform): zlib_tar = self.libs_dir / 'zlib.tar.gz' if self.zlib_version is None: url = latest_zlib_release() - self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION unset, downloading latest from {url}')) else: - url = 'https://zlib.net/fossils/zlib-{}.tar.gz'.format(self.zlib_version) - self.info( - '{:10}: {}'.format('zlib', 'PYXMLSEC_ZLIB_VERSION={}, downloading from {}'.format(self.zlib_version, url)) - ) + url = f'https://zlib.net/fossils/zlib-{self.zlib_version}.tar.gz' + self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION={self.zlib_version}, downloading from {url}')) urlretrieve(url, str(zlib_tar)) # fetch libiconv @@ -324,13 +319,11 @@ def prepare_static_build(self, build_platform): libiconv_tar = self.libs_dir / 'libiconv.tar.gz' if self.libiconv_version is None: url = latest_libiconv_release() - self.info('{:10}: {}'.format('zlib', 'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) else: - url = 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{}.tar.gz'.format(self.libiconv_version) + url = f'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{self.libiconv_version}.tar.gz' self.info( - '{:10}: {}'.format( - 'zlib', 'PYXMLSEC_LIBICONV_VERSION={}, downloading from {}'.format(self.libiconv_version, url) - ) + '{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') ) urlretrieve(url, str(libiconv_tar)) @@ -340,16 +333,12 @@ def prepare_static_build(self, build_platform): self.info('{:10}: {}'.format('libxml2', 'source tar not found, downloading ...')) if self.libxml2_version is None: url = latest_libxml2_release() - self.info('{:10}: {}'.format('libxml2', 'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {url}')) else: version_prefix, _ = self.libxml2_version.rsplit('.', 1) - url = 'https://download.gnome.org/sources/libxml2/{}/libxml2-{}.tar.xz'.format( - version_prefix, self.libxml2_version - ) + url = f'https://download.gnome.org/sources/libxml2/{version_prefix}/libxml2-{self.libxml2_version}.tar.xz' self.info( - '{:10}: {}'.format( - 'libxml2', 'PYXMLSEC_LIBXML2_VERSION={}, downloading from {}'.format(self.libxml2_version, url) - ) + '{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION={self.libxml2_version}, downloading from {url}') ) libxml2_tar = self.libs_dir / 'libxml2.tar.xz' urlretrieve(url, str(libxml2_tar)) @@ -360,16 +349,12 @@ def prepare_static_build(self, build_platform): self.info('{:10}: {}'.format('libxslt', 'source tar not found, downloading ...')) if self.libxslt_version is None: url = latest_libxslt_release() - self.info('{:10}: {}'.format('libxslt', 'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {url}')) else: version_prefix, _ = self.libxslt_version.rsplit('.', 1) - url = 'https://download.gnome.org/sources/libxslt/{}/libxslt-{}.tar.xz'.format( - version_prefix, self.libxslt_version - ) + url = f'https://download.gnome.org/sources/libxslt/{version_prefix}/libxslt-{self.libxslt_version}.tar.xz' self.info( - '{:10}: {}'.format( - 'libxslt', 'PYXMLSEC_LIBXSLT_VERSION={}, downloading from {}'.format(self.libxslt_version, url) - ) + '{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION={self.libxslt_version}, downloading from {url}') ) libxslt_tar = self.libs_dir / 'libxslt.tar.gz' urlretrieve(url, str(libxslt_tar)) @@ -380,26 +365,24 @@ def prepare_static_build(self, build_platform): self.info('{:10}: {}'.format('xmlsec1', 'source tar not found, downloading ...')) if self.xmlsec1_version is None: url = latest_xmlsec_release() - self.info('{:10}: {}'.format('xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {}'.format(url))) + self.info('{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {url}')) else: - url = 'https://github.com/lsh123/xmlsec/releases/download/{v}/xmlsec1-{v}.tar.gz'.format(v=self.xmlsec1_version) + url = f'https://github.com/lsh123/xmlsec/releases/download/{self.xmlsec1_version}/xmlsec1-{self.xmlsec1_version}.tar.gz' self.info( - '{:10}: {}'.format( - 'xmlsec1', 'PYXMLSEC_XMLSEC1_VERSION={}, downloading from {}'.format(self.xmlsec1_version, url) - ) + '{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION={self.xmlsec1_version}, downloading from {url}') ) xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' urlretrieve(url, str(xmlsec1_tar)) for file in (openssl_tar, zlib_tar, libiconv_tar, libxml2_tar, libxslt_tar, xmlsec1_tar): - self.info('Unpacking {}'.format(file.name)) + self.info(f'Unpacking {file.name}') try: with tarfile.open(str(file)) as tar: tar.extractall(path=str(self.build_libs_dir)) - except EOFError: - raise DistutilsError('Bad {} downloaded; remove it and try again.'.format(file.name)) + except EOFError as e: + raise DistutilsError(f'Bad {file.name} downloaded; remove it and try again.') from e - prefix_arg = '--prefix={}'.format(self.prefix_dir) + prefix_arg = f'--prefix={self.prefix_dir}' env = os.environ.copy() cflags = [] @@ -416,14 +399,13 @@ def prepare_static_build(self, build_platform): arch = self.plat_name.rsplit('-', 1)[1] if arch != platform.machine() and arch in ('x86_64', 'arm64'): - self.info('Cross-compiling for {}'.format(arch)) - cflags.append('-arch {}'.format(arch)) - ldflags.append('-arch {}'.format(arch)) + self.info(f'Cross-compiling for {arch}') + cflags.append(f'-arch {arch}') + ldflags.append(f'-arch {arch}') cross_compiling = CrossCompileInfo('darwin64', arch, 'cc') - major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) - if major_version >= 11: - if 'MACOSX_DEPLOYMENT_TARGET' not in env: - env['MACOSX_DEPLOYMENT_TARGET'] = "11.0" + major_version, _ = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11 and 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' env['CFLAGS'] = ' '.join(cflags) env['LDFLAGS'] = ' '.join(ldflags) @@ -437,20 +419,18 @@ def prepare_static_build(self, build_platform): else: openssl_config_cmd.insert(0, './config') subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) - subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env - ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install_sw'], cwd=str(openssl_dir), env=env) self.info('Building zlib') zlib_dir = next(self.build_libs_dir.glob('zlib-*')) subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(zlib_dir), env=env) host_arg = [] if cross_compiling: - host_arg = ['--host={}'.format(cross_compiling.arch)] + host_arg = [f'--host={cross_compiling.arch}'] self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) @@ -465,8 +445,8 @@ def prepare_static_build(self, build_platform): cwd=str(libiconv_dir), env=env, ) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libiconv_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libiconv_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libiconv_dir), env=env) self.info('Building LibXML2') libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) @@ -478,15 +458,15 @@ def prepare_static_build(self, build_platform): '--disable-shared', '--without-lzma', '--without-python', - '--with-iconv={}'.format(self.prefix_dir), - '--with-zlib={}'.format(self.prefix_dir), + f'--with-iconv={self.prefix_dir}', + f'--with-zlib={self.prefix_dir}', *host_arg, ], cwd=str(libxml2_dir), env=env, ) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxml2_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxml2_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxml2_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxml2_dir), env=env) self.info('Building libxslt') libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) @@ -498,14 +478,14 @@ def prepare_static_build(self, build_platform): '--disable-shared', '--without-python', '--without-crypto', - '--with-libxml-prefix={}'.format(self.prefix_dir), + f'--with-libxml-prefix={self.prefix_dir}', *host_arg, ], cwd=str(libxslt_dir), env=env, ) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libxslt_dir), env=env) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(libxslt_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxslt_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxslt_dir), env=env) self.info('Building xmlsec1') ldflags.append('-lpthread') @@ -524,21 +504,24 @@ def prepare_static_build(self, build_platform): '--enable-shared=no', '--enable-static-linking=yes', '--with-default-crypto=openssl', - '--with-openssl={}'.format(self.prefix_dir), - '--with-libxml={}'.format(self.prefix_dir), - '--with-libxslt={}'.format(self.prefix_dir), + f'--with-openssl={self.prefix_dir}', + f'--with-libxml={self.prefix_dir}', + f'--with-libxslt={self.prefix_dir}', *host_arg, ], cwd=str(xmlsec1_dir), env=env, ) + include_flags = [ + f'-I{self.prefix_dir / "include"}', + f'-I{self.prefix_dir / "include" / "libxml"}', + ] subprocess.check_call( - ['make', '-j{}'.format(multiprocessing.cpu_count() + 1)] - + ['-I{}'.format(str(self.prefix_dir / 'include')), '-I{}'.format(str(self.prefix_dir / 'include' / 'libxml'))], + ['make', f'-j{multiprocessing.cpu_count() + 1}', *include_flags], cwd=str(xmlsec1_dir), env=env, ) - subprocess.check_call(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(xmlsec1_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(xmlsec1_dir), env=env) ext = self.ext_map['xmlsec'] ext.define_macros = [ @@ -584,7 +567,7 @@ def prepare_static_build(self, build_platform): setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] -with io.open('README.md', encoding='utf-8') as f: +with open('README.md', encoding='utf-8') as f: long_desc = f.read() @@ -596,10 +579,10 @@ def prepare_static_build(self, build_platform): long_description_content_type='text/markdown', ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, - python_requires='>=3.5', + python_requires='>=3.9', setup_requires=setup_reqs, install_requires=['lxml>=3.8'], - author="Bulat Gaifullin", + author='Bulat Gaifullin', author_email='support@mehcode.com', maintainer='Oleg Hoefling', maintainer_email='oleg.hoefling@gmail.com', @@ -619,12 +602,11 @@ def prepare_static_build(self, build_platform): 'Operating System :: OS Independent', 'Programming Language :: C', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3c3ea94f..a9254ddd 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -1,10 +1,5 @@ import sys -from typing import NamedTuple - -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final +from typing import Final, NamedTuple class __KeyData(NamedTuple): # __KeyData type href: str diff --git a/tests/base.py b/tests/base.py index 48aef81d..1d21c89b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -25,16 +25,16 @@ class TestMemoryLeaks(unittest.TestCase): iterations = test_iterations - data_dir = os.path.join(os.path.dirname(__file__), "data") + data_dir = os.path.join(os.path.dirname(__file__), 'data') def setUp(self): gc.disable() - self.addTypeEqualityFunc(etype, "assertXmlEqual") + self.addTypeEqualityFunc(etype, 'assertXmlEqual') xmlsec.enable_debug_trace(1) def run(self, result=None): # run first time - super(TestMemoryLeaks, self).run(result=result) + super().run(result=result) if self.iterations == 0: return @@ -43,7 +43,7 @@ def run(self, result=None): m_hits = 0 o_hits = 0 for _ in range(self.iterations): - super(TestMemoryLeaks, self).run(result=result) + super().run(result=result) m_usage_n = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss if m_usage_n > m_usage: m_usage = m_usage_n @@ -58,13 +58,13 @@ def run(self, result=None): if m_hits > int(self.iterations * 0.8): result.buffer = False try: - raise AssertionError("memory leak detected") + raise AssertionError('memory leak detected') except AssertionError: result.addError(self, sys.exc_info()) if o_hits > int(self.iterations * 0.8): result.buffer = False try: - raise AssertionError("unreferenced objects detected") + raise AssertionError('unreferenced objects detected') except AssertionError: result.addError(self, sys.exc_info()) @@ -74,7 +74,7 @@ def path(self, name): def load(self, name): """Load resource by name.""" - with open(self.path(name), "rb") as stream: + with open(self.path(name), 'rb') as stream: return stream.read() def load_xml(self, name, xpath=None): @@ -88,28 +88,26 @@ def load_xml(self, name, xpath=None): def dump(self, root): print(etree.tostring(root)) - def assertXmlEqual(self, first, second, msg=None): # noqa: N802 + def assertXmlEqual(self, first, second, msg=None): """Check equality of etree.roots.""" msg = msg or '' if first.tag != second.tag: - self.fail('Tags do not match: {} and {}. {}'.format(first.tag, second.tag, msg)) + self.fail(f'Tags do not match: {first.tag} and {second.tag}. {msg}') for name, value in first.attrib.items(): if second.attrib.get(name) != value: - self.fail('Attributes do not match: {}={!r}, {}={!r}. {}'.format(name, value, name, second.attrib.get(name), msg)) - for name in second.attrib.keys(): + self.fail(f'Attributes do not match: {name}={value!r}, {name}={second.attrib.get(name)!r}. {msg}') + for name in second.attrib: if name not in first.attrib: - self.fail('x2 has an attribute x1 is missing: {}. {}'.format(name, msg)) + self.fail(f'x2 has an attribute x1 is missing: {name}. {msg}') if not _xml_text_compare(first.text, second.text): - self.fail('text: {!r} != {!r}. {}'.format(first.text, second.text, msg)) + self.fail(f'text: {first.text!r} != {second.text!r}. {msg}') if not _xml_text_compare(first.tail, second.tail): - self.fail('tail: {!r} != {!r}. {}'.format(first.tail, second.tail, msg)) + self.fail(f'tail: {first.tail!r} != {second.tail!r}. {msg}') cl1 = sorted(first.getchildren(), key=lambda x: x.tag) cl2 = sorted(second.getchildren(), key=lambda x: x.tag) if len(cl1) != len(cl2): - self.fail('children length differs, {} != {}. {}'.format(len(cl1), len(cl2), msg)) - i = 0 + self.fail(f'children length differs, {len(cl1)} != {len(cl2)}. {msg}') for c1, c2 in zip(cl1, cl2): - i += 1 self.assertXmlEqual(c1, c2) diff --git a/tests/conftest.py b/tests/conftest.py index 675258c5..a65235d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ def pytest_collection_modifyitems(items): - """ - Put the module init test first. + """Put the module init test first. This way, we implicitly check whether any subsequent test fails because of module reinitialization. """ diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 247f1b18..3f5076d2 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -24,7 +24,7 @@ def find_alts(component_name, alts) -> str: for a in alts: if os.path.exists(a): return a - raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) + raise unittest.SkipTest(f'Required component is missing: {component_name}') def run_cmd(args, softhsm_conf=None): @@ -101,7 +101,7 @@ def run_cmd(args, softhsm_conf=None): def _temp_file() -> str: - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(delete=False) # noqa: SIM115 p11_test_files.append(f.name) return f.name @@ -123,24 +123,20 @@ def setup() -> None: if softhsm_version == 2: softhsm_db = _temp_dir() f.write( - """ + f""" # Generated by test -directories.tokendir = {} +directories.tokendir = {softhsm_db} objectstore.backend = file log.level = DEBUG -""".format( - softhsm_db - ) +""" ) else: softhsm_db = _temp_file() f.write( - """ + f""" # Generated by test -0:{} -""".format( - softhsm_db - ) +0:{softhsm_db} +""" ) logging.debug('Initializing the token') @@ -227,13 +223,13 @@ def setup() -> None: '[pkcs11_section]', 'engine_id = pkcs11', # dynamic_path, - "MODULE_PATH = {}".format(component_path['P11_MODULE']), + 'MODULE_PATH = {}'.format(component_path['P11_MODULE']), 'init = 0', ] ) ) - with open(openssl_conf, 'r') as f: + with open(openssl_conf) as f: logging.debug('-------- START DEBUG openssl_conf --------') logging.debug(f.readlines()) logging.debug('-------- END DEBUG openssl_conf --------') @@ -309,7 +305,7 @@ def setup() -> None: softhsm_conf=softhsm_conf, ) - # TODO: Should be teardowned in teardown # noqa: T101 + # TODO: Should be teardowned in teardown os.environ['SOFTHSM_CONF'] = softhsm_conf os.environ['SOFTHSM2_CONF'] = softhsm_conf diff --git a/tests/test_constants.py b/tests/test_constants.py index f79d19f0..2c39d5f6 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -21,22 +21,22 @@ def _constants(typename): @pytest.mark.parametrize('transform', _constants('__Transform'), ids=repr) def test_transform_str(transform): """Test string representation of ``xmlsec.constants.__Transform``.""" - assert str(transform) == '{}, {}'.format(transform.name, transform.href) + assert str(transform) == f'{transform.name}, {transform.href}' @pytest.mark.parametrize('transform', _constants('__Transform'), ids=repr) def test_transform_repr(transform): """Test raw string representation of ``xmlsec.constants.__Transform``.""" - assert repr(transform) == '__Transform({!r}, {!r}, {})'.format(transform.name, transform.href, transform.usage) + assert repr(transform) == f'__Transform({transform.name!r}, {transform.href!r}, {transform.usage})' @pytest.mark.parametrize('keydata', _constants('__KeyData'), ids=repr) def test_keydata_str(keydata): """Test string representation of ``xmlsec.constants.__KeyData``.""" - assert str(keydata) == '{}, {}'.format(keydata.name, keydata.href) + assert str(keydata) == f'{keydata.name}, {keydata.href}' @pytest.mark.parametrize('keydata', _constants('__KeyData'), ids=repr) def test_keydata_repr(keydata): """Test raw string representation of ``xmlsec.constants.__KeyData``.""" - assert repr(keydata) == '__KeyData({!r}, {!r})'.format(keydata.name, keydata.href) + assert repr(keydata) == f'__KeyData({keydata.name!r}, {keydata.href!r})' diff --git a/tests/test_doc_examples.py b/tests/test_doc_examples.py index 2fc490f3..7aa8e517 100644 --- a/tests/test_doc_examples.py +++ b/tests/test_doc_examples.py @@ -3,24 +3,17 @@ import contextlib import os import runpy -import sys +from pathlib import Path import pytest -if sys.version_info >= (3, 4): - from pathlib import Path -else: # python2.7 compat - from _pytest.pathlib import Path - - examples_dir = Path(__file__, '../../doc/source/examples').resolve() examples = sorted(examples_dir.glob('*.py')) @contextlib.contextmanager def cd(where_to): - """ - Temporarily change the working directory. + """Temporarily change the working directory. Restore the current working dir after exiting the context. """ @@ -34,8 +27,7 @@ def cd(where_to): @pytest.mark.parametrize('example', examples, ids=lambda p: p.name) def test_doc_example(example): - """ - Verify example scripts included in the docs are up to date. + """Verify example scripts included in the docs are up to date. Execute each script in :file:`docs/source/examples`, not raising any errors is good enough. diff --git a/tests/test_ds.py b/tests/test_ds.py index 38f0b25c..dd0657d3 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -25,13 +25,13 @@ def test_no_key(self): def test_del_key(self): ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) del ctx.key self.assertIsNone(ctx.key) def test_set_key(self): ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) def test_set_key_bad_type(self): @@ -46,9 +46,9 @@ def test_set_invalid_key(self): def test_register_id(self): ctx = xmlsec.SignatureContext() - root = self.load_xml("sign_template.xml") - sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, "Id") - ctx.register_id(sign, "Id") + root = self.load_xml('sign_template.xml') + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, 'Id') + ctx.register_id(sign, 'Id') def test_register_id_bad_args(self): ctx = xmlsec.SignatureContext() @@ -57,41 +57,41 @@ def test_register_id_bad_args(self): def test_register_id_with_namespace_without_attribute(self): ctx = xmlsec.SignatureContext() - root = self.load_xml("sign_template.xml") - sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, "Id") + root = self.load_xml('sign_template.xml') + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, 'Id') with self.assertRaisesRegex(xmlsec.Error, 'missing attribute.'): - ctx.register_id(sign, "Id", id_ns='foo') + ctx.register_id(sign, 'Id', id_ns='foo') def test_sign_bad_args(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaises(TypeError): ctx.sign('') def test_sign_fail(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): ctx.sign(self.load_xml('sign1-in.xml')) def test_sign_case1(self): """Should sign a pre-constructed template file using a key from a PEM file.""" - root = self.load_xml("sign1-in.xml") + root = self.load_xml('sign1-in.xml') sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign1-out.xml"), root) + self.assertEqual(self.load_xml('sign1-out.xml'), root) def test_sign_case2(self): """Should sign a dynamicaly constructed template file using a key from a PEM file.""" - root = self.load_xml("sign2-in.xml") + root = self.load_xml('sign2-in.xml') sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1) self.assertIsNotNone(sign) root.append(sign) @@ -101,17 +101,17 @@ def test_sign_case2(self): xmlsec.template.add_key_name(ki) ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign2-out.xml"), root) + self.assertEqual(self.load_xml('sign2-out.xml'), root) def test_sign_case3(self): """Should sign a file using a dynamicaly created template, key from PEM and an X509 cert.""" - root = self.load_xml("sign3-in.xml") + root = self.load_xml('sign3-in.xml') sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1) self.assertIsNotNone(sign) root.append(sign) @@ -121,23 +121,23 @@ def test_sign_case3(self): xmlsec.template.add_x509_data(ki) ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.load_cert_from_file(self.path('rsacert.pem'), consts.KeyDataFormatPem) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign3-out.xml"), root) + self.assertEqual(self.load_xml('sign3-out.xml'), root) def test_sign_case4(self): """Should sign a file using a dynamically created template, key from PEM and an X509 cert with custom ns.""" - root = self.load_xml("sign4-in.xml") - xmlsec.tree.add_ids(root, ["ID"]) + root = self.load_xml('sign4-in.xml') + xmlsec.tree.add_ids(root, ['ID']) elem_id = root.get('ID', None) if elem_id: elem_id = '#' + elem_id - sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns="ds") + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') self.assertIsNotNone(sign) root.append(sign) ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri=elem_id) @@ -147,18 +147,18 @@ def test_sign_case4(self): xmlsec.template.add_x509_data(ki) ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.load_cert_from_file(self.path('rsacert.pem'), consts.KeyDataFormatPem) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign4-out.xml"), root) + self.assertEqual(self.load_xml('sign4-out.xml'), root) def test_sign_case5(self): """Should sign a file using a dynamicaly created template, key from PEM file and an X509 certificate.""" - root = self.load_xml("sign5-in.xml") + root = self.load_xml('sign5-in.xml') sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1) self.assertIsNotNone(sign) root.append(sign) @@ -175,11 +175,11 @@ def test_sign_case5(self): xmlsec.template.x509_issuer_serial_add_serial_number(x509_issuer_serial, '1') ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.load_cert_from_file(self.path('rsacert.pem'), consts.KeyDataFormatPem) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) if (1, 2, 36) <= xmlsec.get_libxmlsec_version() <= (1, 2, 37): @@ -190,7 +190,7 @@ def test_sign_case5(self): def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaises(TypeError): ctx.sign_binary(bytes=1, transform='') @@ -202,23 +202,23 @@ def test_sign_binary_no_key(self): @unittest.skipIf(not hasattr(consts, 'TransformXslt'), reason='XSLT transformations not enabled') def test_sign_binary_invalid_signature_method(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaisesRegex(xmlsec.Error, 'incompatible signature method'): ctx.sign_binary(bytes=b'', transform=consts.TransformXslt) def test_sign_binary(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) - sign = ctx.sign_binary(self.load("sign6-in.bin"), consts.TransformRsaSha1) - self.assertEqual(self.load("sign6-out.bin"), sign) + sign = ctx.sign_binary(self.load('sign6-in.bin'), consts.TransformRsaSha1) + self.assertEqual(self.load('sign6-out.bin'), sign) def test_sign_binary_twice_not_possible(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) data = self.load('sign6-in.bin') ctx.sign_binary(data, consts.TransformRsaSha1) with self.assertRaisesRegex(xmlsec.Error, 'Signature context already used; it is designed for one use only.'): @@ -226,13 +226,13 @@ def test_sign_binary_twice_not_possible(self): def test_verify_bad_args(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaises(TypeError): ctx.verify('') def test_verify_fail(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaisesRegex(xmlsec.Error, 'failed to verify'): ctx.verify(self.load_xml('sign1-in.xml')) @@ -252,41 +252,41 @@ def test_verify_case_5(self): self.check_verify(5) def check_verify(self, i): - root = self.load_xml("sign{}-out.xml".format(i)) - xmlsec.tree.add_ids(root, ["ID"]) + root = self.load_xml(f'sign{i}-out.xml') + xmlsec.tree.add_ids(root, ['ID']) sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) - self.assertEqual(consts.NodeSignature, sign.tag.partition("}")[2]) + self.assertEqual(consts.NodeSignature, sign.tag.partition('}')[2]) ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsapub.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsapub.pem' - self.assertEqual("rsapub.pem", ctx.key.name) + self.assertEqual('rsapub.pem', ctx.key.name) ctx.verify(sign) def test_validate_binary_sign(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) - ctx.verify_binary(self.load("sign6-in.bin"), consts.TransformRsaSha1, self.load("sign6-out.bin")) + ctx.verify_binary(self.load('sign6-in.bin'), consts.TransformRsaSha1, self.load('sign6-out.bin')) def test_validate_binary_sign_fail(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) with self.assertRaises(xmlsec.Error): - ctx.verify_binary(self.load("sign6-in.bin"), consts.TransformRsaSha1, b"invalid") + ctx.verify_binary(self.load('sign6-in.bin'), consts.TransformRsaSha1, b'invalid') def test_enable_reference_transform(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) ctx.enable_reference_transform(consts.TransformRsaSha1) def test_enable_reference_transform_bad_args(self): @@ -301,7 +301,7 @@ def test_enable_reference_transform_bad_args(self): def test_enable_signature_transform(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) ctx.enable_signature_transform(consts.TransformRsaSha1) def test_enable_signature_transform_bad_args(self): @@ -316,12 +316,12 @@ def test_enable_signature_transform_bad_args(self): def test_set_enabled_key_data(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) ctx.set_enabled_key_data([consts.KeyDataAes]) def test_set_enabled_key_data_empty(self): ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) ctx.set_enabled_key_data([]) def test_set_enabled_key_data_bad_args(self): diff --git a/tests/test_enc.py b/tests/test_enc.py index 1788b4d6..41f78d74 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -28,18 +28,18 @@ def test_no_key(self): def test_get_key(self): ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) self.assertIsNone(ctx.key) - ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) + ctx.key = xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem) self.assertIsNotNone(ctx.key) def test_del_key(self): ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) - ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) + ctx.key = xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem) del ctx.key self.assertIsNone(ctx.key) def test_set_key(self): ctx = xmlsec.EncryptionContext(manager=xmlsec.KeysManager()) - ctx.key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) + ctx.key = xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem) self.assertIsNotNone(ctx.key) def test_set_key_bad_type(self): @@ -54,16 +54,16 @@ def test_set_invalid_key(self): def test_encrypt_xml(self): root = self.load_xml('enc1-in.xml') - enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns="xenc") + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns='xenc') xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") + ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) xmlsec.template.encrypted_data_ensure_cipher_value(ek) data = root.find('./Data') self.assertIsNotNone(data) manager = xmlsec.KeysManager() - manager.add_key(xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem)) + manager.add_key(xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem)) ctx = xmlsec.EncryptionContext(manager) ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) @@ -73,12 +73,12 @@ def test_encrypt_xml(self): enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#aes128-cbc", enc_method.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#aes128-cbc', enc_method.get('Algorithm')) ki = xmlsec.tree.find_child(enc_data, consts.NodeKeyInfo, consts.DSigNs) self.assertIsNotNone(ki) enc_method2 = xmlsec.tree.find_node(ki, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method2) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p", enc_method2.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p', enc_method2.get('Algorithm')) cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) @@ -109,32 +109,32 @@ def test_encrypt_xml_fail(self): def test_encrypt_binary(self): root = self.load_xml('enc2-in.xml') enc_data = xmlsec.template.encrypted_data_create( - root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns="xenc", mime_type="binary/octet-stream" + root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc', mime_type='binary/octet-stream' ) xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") + ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) xmlsec.template.encrypted_data_ensure_cipher_value(ek) manager = xmlsec.KeysManager() - manager.add_key(xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem)) + manager.add_key(xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem)) ctx = xmlsec.EncryptionContext(manager) ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) encrypted = ctx.encrypt_binary(enc_data, b'test') self.assertIsNotNone(encrypted) - self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#aes128-cbc", enc_method.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#aes128-cbc', enc_method.get('Algorithm')) ki = xmlsec.tree.find_child(enc_data, consts.NodeKeyInfo, consts.DSigNs) self.assertIsNotNone(ki) enc_method2 = xmlsec.tree.find_node(ki, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method2) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p", enc_method2.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p', enc_method2.get('Algorithm')) cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) @@ -151,15 +151,15 @@ def test_encrypt_binary_bad_template(self): def test_encrypt_uri(self): root = self.load_xml('enc2-in.xml') enc_data = xmlsec.template.encrypted_data_create( - root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns="xenc", mime_type="binary/octet-stream" + root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc', mime_type='binary/octet-stream' ) xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns="dsig") + ki = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) xmlsec.template.encrypted_data_ensure_cipher_value(ek) manager = xmlsec.KeysManager() - manager.add_key(xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem)) + manager.add_key(xmlsec.Key.from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatCertPem)) ctx = xmlsec.EncryptionContext(manager) ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) @@ -169,17 +169,17 @@ def test_encrypt_uri(self): encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) self.assertIsNotNone(encrypted) - self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#aes128-cbc", enc_method.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#aes128-cbc', enc_method.get('Algorithm')) ki = xmlsec.tree.find_child(enc_data, consts.NodeKeyInfo, consts.DSigNs) self.assertIsNotNone(ki) enc_method2 = xmlsec.tree.find_node(ki, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method2) - self.assertEqual("http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p", enc_method2.get("Algorithm")) + self.assertEqual('http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p', enc_method2.get('Algorithm')) cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) @@ -205,7 +205,7 @@ def test_decrypt_key(self): self.assertIsNotNone(enc_key) manager = xmlsec.KeysManager() - manager.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + manager.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) ctx = xmlsec.EncryptionContext(manager) keydata = ctx.decrypt(enc_key) ctx.reset() @@ -215,19 +215,19 @@ def test_decrypt_key(self): self.assertIsNotNone(enc_data) decrypted = ctx.decrypt(enc_data) self.assertIsNotNone(decrypted) - self.assertEqual(self.load_xml("enc3-in.xml"), decrypted) + self.assertEqual(self.load_xml('enc3-in.xml'), decrypted) def check_decrypt(self, i): - root = self.load_xml('enc{}-out.xml'.format(i)) + root = self.load_xml(f'enc{i}-out.xml') enc_data = xmlsec.tree.find_child(root, consts.NodeEncryptedData, consts.EncNs) self.assertIsNotNone(enc_data) manager = xmlsec.KeysManager() - manager.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + manager.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) ctx = xmlsec.EncryptionContext(manager) decrypted = ctx.decrypt(enc_data) self.assertIsNotNone(decrypted) - self.assertEqual(self.load_xml("enc{}-in.xml".format(i)), root) + self.assertEqual(self.load_xml(f'enc{i}-in.xml'), root) def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() diff --git a/tests/test_keys.py b/tests/test_keys.py index 0d41abef..977ddf82 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -9,33 +9,32 @@ class TestKeys(base.TestMemoryLeaks): def test_key_from_memory(self): - key = xmlsec.Key.from_memory(self.load("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_memory(self.load('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) def test_key_from_memory_with_bad_args(self): with self.assertRaises(TypeError): - xmlsec.Key.from_memory(1, format="") + xmlsec.Key.from_memory(1, format='') def test_key_from_memory_invalid_data(self): with self.assertRaisesRegex(xmlsec.Error, '.*cannot load key.*'): xmlsec.Key.from_memory(b'foo', format=consts.KeyDataFormatPem) def test_key_from_file(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) def test_key_from_file_with_bad_args(self): with self.assertRaises(TypeError): - xmlsec.Key.from_file(1, format="") + xmlsec.Key.from_file(1, format='') def test_key_from_invalid_file(self): - with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(b'foo') - xmlsec.Key.from_file(tmpfile.name, format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'), tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + xmlsec.Key.from_file(tmpfile.name, format=consts.KeyDataFormatPem) def test_key_from_fileobj(self): - with open(self.path("rsakey.pem"), "rb") as fobj: + with open(self.path('rsakey.pem'), 'rb') as fobj: key = xmlsec.Key.from_file(fobj, format=consts.KeyDataFormatPem) self.assertIsNotNone(key) @@ -51,71 +50,69 @@ def test_generate(self): def test_generate_with_bad_args(self): with self.assertRaises(TypeError): - xmlsec.Key.generate(klass="", size="", type="") + xmlsec.Key.generate(klass='', size='', type='') def test_generate_invalid_size(self): with self.assertRaisesRegex(xmlsec.Error, '.*cannot generate key.*'): xmlsec.Key.generate(klass=consts.KeyDataAes, size=0, type=consts.KeyDataTypeSession) def test_from_binary_file(self): - key = xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=self.path("deskey.bin")) + key = xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=self.path('deskey.bin')) self.assertIsNotNone(key) def test_from_binary_file_with_bad_args(self): with self.assertRaises(TypeError): - xmlsec.Key.from_binary_file(klass="", filename=1) + xmlsec.Key.from_binary_file(klass='', filename=1) def test_from_invalid_binary_file(self): - with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(b'foo') - xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=tmpfile.name) + with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'), tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + xmlsec.Key.from_binary_file(klass=consts.KeyDataDes, filename=tmpfile.name) def test_from_binary_data(self): - key = xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=self.load("deskey.bin")) + key = xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=self.load('deskey.bin')) self.assertIsNotNone(key) def test_from_binary_data_with_bad_args(self): with self.assertRaises(TypeError): - xmlsec.Key.from_binary_data(klass="", data=1) + xmlsec.Key.from_binary_data(klass='', data=1) def test_from_invalid_binary_data(self): with self.assertRaisesRegex(xmlsec.Error, '.*cannot read key.*'): xmlsec.Key.from_binary_data(klass=consts.KeyDataDes, data=b'') def test_load_cert_from_file(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - key.load_cert_from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatPem) + key.load_cert_from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatPem) def test_load_cert_from_file_with_bad_args(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) with self.assertRaises(TypeError): - key.load_cert_from_file(1, format="") + key.load_cert_from_file(1, format='') def test_load_cert_from_invalid_file(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(b'foo') - key.load_cert_from_file(tmpfile.name, format=consts.KeyDataFormatPem) + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'), tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + key.load_cert_from_file(tmpfile.name, format=consts.KeyDataFormatPem) def test_load_cert_from_fileobj(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - with open(self.path("rsacert.pem"), "rb") as fobj: + with open(self.path('rsacert.pem'), 'rb') as fobj: key.load_cert_from_file(fobj, format=consts.KeyDataFormatPem) def test_load_cert_from_fileobj_with_bad_args(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - with self.assertRaises(TypeError), open(self.path("rsacert.pem"), "rb") as fobj: + with self.assertRaises(TypeError), open(self.path('rsacert.pem'), 'rb') as fobj: key.load_cert_from_file(fobj, format='') def test_load_cert_from_invalid_fileobj(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) with tempfile.NamedTemporaryFile(delete=False) as tmpfile: tmpfile.write(b'foo') @@ -123,41 +120,41 @@ def test_load_cert_from_invalid_fileobj(self): key.load_cert_from_file(fp, format=consts.KeyDataFormatPem) def test_load_cert_from_memory(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) - key.load_cert_from_memory(self.load("rsacert.pem"), format=consts.KeyDataFormatPem) + key.load_cert_from_memory(self.load('rsacert.pem'), format=consts.KeyDataFormatPem) def test_load_cert_from_memory_with_bad_args(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) with self.assertRaises(TypeError): - key.load_cert_from_memory(1, format="") + key.load_cert_from_memory(1, format='') def test_load_cert_from_memory_invalid_data(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNotNone(key) with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): key.load_cert_from_memory(b'', format=consts.KeyDataFormatPem) def test_get_name(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) self.assertIsNone(key.name) def test_get_name_invalid_key(self): key = xmlsec.Key() with self.assertRaisesRegex(ValueError, 'key is not ready'): - key.name + key.name # noqa: B018 def test_del_name(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) - key.name = "rsakey" + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + key.name = 'rsakey' del key.name self.assertIsNone(key.name) def test_set_name(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) - key.name = "rsakey" - self.assertEqual("rsakey", key.name) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + key.name = 'rsakey' + self.assertEqual('rsakey', key.name) def test_set_name_invalid_key(self): key = xmlsec.Key() @@ -165,56 +162,55 @@ def test_set_name_invalid_key(self): key.name = 'foo' def test_copy(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) key2 = copy.copy(key) del key - key2.load_cert_from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatPem) + key2.load_cert_from_file(self.path('rsacert.pem'), format=consts.KeyDataFormatPem) class TestKeysManager(base.TestMemoryLeaks): def test_add_key(self): - key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) mngr = xmlsec.KeysManager() mngr.add_key(key) def test_add_key_with_bad_args(self): mngr = xmlsec.KeysManager() with self.assertRaises(TypeError): - mngr.add_key("") + mngr.add_key('') def test_load_cert(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) - mngr.load_cert(self.path("rsacert.pem"), format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) + mngr.load_cert(self.path('rsacert.pem'), format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) def test_load_cert_with_bad_args(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) - with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): - with tempfile.NamedTemporaryFile() as tmpfile: - tmpfile.write(b'foo') - mngr.load_cert(tmpfile.name, format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) + with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'), tempfile.NamedTemporaryFile() as tmpfile: + tmpfile.write(b'foo') + mngr.load_cert(tmpfile.name, format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) def test_load_invalid_cert(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) with self.assertRaises(TypeError): - mngr.load_cert(1, format="", type="") + mngr.load_cert(1, format='', type='') def test_load_cert_from_memory(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) - mngr.load_cert_from_memory(self.load("rsacert.pem"), format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) + mngr.load_cert_from_memory(self.load('rsacert.pem'), format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) def test_load_cert_from_memory_with_bad_args(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) with self.assertRaises(TypeError): - mngr.load_cert_from_memory(1, format="", type="") + mngr.load_cert_from_memory(1, format='', type='') def test_load_cert_from_memory_invalid_data(self): mngr = xmlsec.KeysManager() - mngr.add_key(xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem)) + mngr.add_key(xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)) with self.assertRaisesRegex(xmlsec.Error, '.*cannot load cert.*'): mngr.load_cert_from_memory(b'', format=consts.KeyDataFormatPem, type=consts.KeyDataTypeTrusted) diff --git a/tests/test_main.py b/tests/test_main.py index 3db18582..8f1501f2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -10,7 +10,7 @@ class TestBase64LineSize(base.TestMemoryLeaks): def tearDown(self): xmlsec.base64_default_line_size(64) - super(TestBase64LineSize, self).tearDown() + super().tearDown() def test_get_base64_default_line_size(self): self.assertEqual(xmlsec.base64_default_line_size(), 64) @@ -43,12 +43,12 @@ def setUp(self): xmlsec.cleanup_callbacks() def _sign_doc(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) - xmlsec.template.add_reference(sign, consts.TransformSha1, uri="cid:123456") + xmlsec.template.add_reference(sign, consts.TransformSha1, uri='cid:123456') ctx = xmlsec.SignatureContext() - ctx.key = xmlsec.Key.from_file(self.path("rsakey.pem"), format=consts.KeyDataFormatPem) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) ctx.sign(sign) return sign @@ -78,7 +78,7 @@ def _register_match_callbacks(self): def _find(self, elem, *tags): try: return elem.xpath( - './' + '/'.join('xmldsig:{}'.format(tag) for tag in tags), + './' + '/'.join(f'xmldsig:{tag}' for tag in tags), namespaces={ 'xmldsig': 'http://www.w3.org/2000/09/xmldsig#', }, @@ -125,7 +125,7 @@ def match_cb(filename): self._verify_external_data_signature() self.assertEqual(bad_match_calls, 0) - @skipIf(sys.platform == "win32", "unclear behaviour on windows") + @skipIf(sys.platform == 'win32', 'unclear behaviour on windows') def test_failed_sign_because_default_callbacks(self): mismatch_calls = 0 diff --git a/tests/test_pkcs11.py b/tests/test_pkcs11.py index accd29ae..cba1a3f0 100644 --- a/tests/test_pkcs11.py +++ b/tests/test_pkcs11.py @@ -2,7 +2,7 @@ from tests import base from xmlsec import constants as consts -KEY_URL = "pkcs11;pkcs11:token=test;object=test;pin-value=secret1" +KEY_URL = 'pkcs11;pkcs11:token=test;object=test;pin-value=secret1' def setUpModule(): @@ -43,7 +43,7 @@ def test_sign_fail(self): def test_sign_case1(self): """Should sign a pre-constructed template file using a key from a pkcs11 engine.""" - root = self.load_xml("sign1-in.xml") + root = self.load_xml('sign1-in.xml') sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) @@ -51,7 +51,7 @@ def test_sign_case1(self): ctx.key = xmlsec.Key.from_engine(KEY_URL) self.assertIsNotNone(ctx.key) ctx.key.name = 'rsakey.pem' - self.assertEqual("rsakey.pem", ctx.key.name) + self.assertEqual('rsakey.pem', ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign1-out.xml"), root) + self.assertEqual(self.load_xml('sign1-out.xml'), root) diff --git a/tests/test_templates.py b/tests/test_templates.py index 3bae7e55..bbf7f42d 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -10,61 +10,61 @@ class TestTemplates(base.TestMemoryLeaks): def test_create(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create( - root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1, id="Id", ns="test" + root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1, id='Id', ns='test' ) - self.assertEqual("Id", sign.get("Id")) - self.assertEqual("test", sign.prefix) + self.assertEqual('Id', sign.get('Id')) + self.assertEqual('test', sign.prefix) def test_create_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.create('', c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) def test_encrypt_data_create(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') enc = xmlsec.template.encrypted_data_create( - root, method=consts.TransformDes3Cbc, id="Id", type="Type", mime_type="MimeType", encoding="Encoding", ns="test" + root, method=consts.TransformDes3Cbc, id='Id', type='Type', mime_type='MimeType', encoding='Encoding', ns='test' ) - for a in ("Id", "Type", "MimeType", "Encoding"): + for a in ('Id', 'Type', 'MimeType', 'Encoding'): self.assertEqual(a, enc.get(a)) - self.assertEqual("test", enc.prefix) + self.assertEqual('test', enc.prefix) def test_ensure_key_info(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) - ki = xmlsec.template.ensure_key_info(sign, id="Id") - self.assertEqual("Id", ki.get("Id")) + ki = xmlsec.template.ensure_key_info(sign, id='Id') + self.assertEqual('Id', ki.get('Id')) def test_ensure_key_info_fail(self): with self.assertRaisesRegex(xmlsec.Error, 'cannot ensure key info.'): - xmlsec.template.ensure_key_info(etree.fromstring(b''), id="Id") + xmlsec.template.ensure_key_info(etree.fromstring(b''), id='Id') def test_ensure_key_info_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.ensure_key_info('', id=0) def test_add_encrypted_key(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) ek = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep) - self.assertEqual(ek, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeEncryptedKey, consts.EncNs)) - ek2 = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep, id="Id", type="Type", recipient="Recipient") - for a in ("Id", "Type", "Recipient"): + self.assertEqual(ek, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeEncryptedKey, consts.EncNs)) + ek2 = xmlsec.template.add_encrypted_key(ki, consts.TransformRsaOaep, id='Id', type='Type', recipient='Recipient') + for a in ('Id', 'Type', 'Recipient'): self.assertEqual(a, ek2.get(a)) def test_add_key_name(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) kn = xmlsec.template.add_key_name(ki) - self.assertEqual(kn, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyName, consts.DSigNs)) - kn2 = xmlsec.template.add_key_name(ki, name="name") - self.assertEqual("name", kn2.text) + self.assertEqual(kn, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeKeyName, consts.DSigNs)) + kn2 = xmlsec.template.add_key_name(ki, name='name') + self.assertEqual('name', kn2.text) def test_add_key_name_none(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) kn2 = xmlsec.template.add_key_name(ki, name=None) @@ -76,10 +76,10 @@ def test_add_key_name_bad_args(self): xmlsec.template.add_key_name('') def test_add_reference(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) - ref = xmlsec.template.add_reference(sign, consts.TransformSha1, id="Id", uri="URI", type="Type") - for a in ("Id", "URI", "Type"): + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, id='Id', uri='URI', type='Type') + for a in ('Id', 'URI', 'Type'): self.assertEqual(a, ref.get(a)) def test_add_reference_bad_args(self): @@ -99,18 +99,18 @@ def test_add_transform_bad_args(self): xmlsec.template.add_transform(etree.Element('root'), '') def test_add_key_value(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) kv = xmlsec.template.add_key_value(ki) - self.assertEqual(kv, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeKeyValue, consts.DSigNs)) + self.assertEqual(kv, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeKeyValue, consts.DSigNs)) def test_add_key_value_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.add_key_value('') def test_add_x509_data(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) x509 = xmlsec.template.add_x509_data(ki) @@ -121,22 +121,22 @@ def test_add_x509_data(self): xmlsec.template.x509_data_add_subject_name(x509) xmlsec.template.x509_issuer_serial_add_issuer_name(issuer) xmlsec.template.x509_issuer_serial_add_serial_number(issuer) - self.assertEqual(x509, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeX509Data, consts.DSigNs)) + self.assertEqual(x509, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeX509Data, consts.DSigNs)) def test_add_x509_data_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.add_x509_data('') def test_x509_issuer_serial_add_issuer(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ki = xmlsec.template.ensure_key_info(sign) x509 = xmlsec.template.add_x509_data(ki) issuer = xmlsec.template.x509_data_add_issuer_serial(x509) - name = xmlsec.template.x509_issuer_serial_add_issuer_name(issuer, name="Name") - serial = xmlsec.template.x509_issuer_serial_add_serial_number(issuer, serial="Serial") - self.assertEqual("Name", name.text) - self.assertEqual("Serial", serial.text) + name = xmlsec.template.x509_issuer_serial_add_issuer_name(issuer, name='Name') + serial = xmlsec.template.x509_issuer_serial_add_serial_number(issuer, serial='Serial') + self.assertEqual('Name', name.text) + self.assertEqual('Serial', serial.text) def test_x509_issuer_serial_add_issuer_bad_args(self): with self.assertRaises(TypeError): @@ -175,23 +175,23 @@ def test_encrypted_data_create_bad_args(self): xmlsec.template.encrypted_data_create('', 0) def test_encrypted_data_ensure_cipher_value(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc) cv = xmlsec.template.encrypted_data_ensure_cipher_value(enc) - self.assertEqual(cv, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeCipherValue, consts.EncNs)) + self.assertEqual(cv, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeCipherValue, consts.EncNs)) def test_encrypted_data_ensure_cipher_value_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.encrypted_data_ensure_cipher_value('') def test_encrypted_data_ensure_key_info(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc) ki = xmlsec.template.encrypted_data_ensure_key_info(enc) - self.assertEqual(ki, xmlsec.tree.find_node(self.load_xml("enc_template.xml"), consts.NodeKeyInfo, consts.DSigNs)) - ki2 = xmlsec.template.encrypted_data_ensure_key_info(enc, id="Id", ns="test") - self.assertEqual("Id", ki2.get("Id")) - self.assertEqual("test", ki2.prefix) + self.assertEqual(ki, xmlsec.tree.find_node(self.load_xml('enc_template.xml'), consts.NodeKeyInfo, consts.DSigNs)) + ki2 = xmlsec.template.encrypted_data_ensure_key_info(enc, id='Id', ns='test') + self.assertEqual('Id', ki2.get('Id')) + self.assertEqual('test', ki2.prefix) def test_encrypted_data_ensure_key_info_bad_args(self): with self.assertRaises(TypeError): @@ -199,14 +199,14 @@ def test_encrypted_data_ensure_key_info_bad_args(self): @unittest.skipIf(not hasattr(consts, 'TransformXslt'), reason='XSLT transformations not enabled') def test_transform_add_c14n_inclusive_namespaces(self): - root = self.load_xml("doc.xml") + root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) ref = xmlsec.template.add_reference(sign, consts.TransformSha1) trans1 = xmlsec.template.add_transform(ref, consts.TransformEnveloped) - xmlsec.template.transform_add_c14n_inclusive_namespaces(trans1, "default") + xmlsec.template.transform_add_c14n_inclusive_namespaces(trans1, 'default') trans2 = xmlsec.template.add_transform(ref, consts.TransformXslt) - xmlsec.template.transform_add_c14n_inclusive_namespaces(trans2, ["ns1", "ns2"]) - self.assertEqual(ref, xmlsec.tree.find_node(self.load_xml("sign_template.xml"), consts.NodeReference, consts.DSigNs)) + xmlsec.template.transform_add_c14n_inclusive_namespaces(trans2, ['ns1', 'ns2']) + self.assertEqual(ref, xmlsec.tree.find_node(self.load_xml('sign_template.xml'), consts.NodeReference, consts.DSigNs)) def test_transform_add_c14n_inclusive_namespaces_bad_args(self): with self.assertRaises(TypeError): diff --git a/tests/test_tree.py b/tests/test_tree.py index 4c79c8de..5e80a60a 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -6,7 +6,7 @@ class TestTree(base.TestMemoryLeaks): def test_find_child(self): - root = self.load_xml("sign_template.xml") + root = self.load_xml('sign_template.xml') si = xmlsec.tree.find_child(root, consts.NodeSignedInfo, consts.DSigNs) self.assertEqual(consts.NodeSignedInfo, si.tag.partition('}')[2]) self.assertIsNone(xmlsec.tree.find_child(root, consts.NodeReference)) @@ -17,7 +17,7 @@ def test_find_child_bad_args(self): xmlsec.tree.find_child('', 0, True) def test_find_parent(self): - root = self.load_xml("sign_template.xml") + root = self.load_xml('sign_template.xml') si = xmlsec.tree.find_child(root, consts.NodeSignedInfo, consts.DSigNs) self.assertIs(root, xmlsec.tree.find_parent(si, consts.NodeSignature)) self.assertIsNone(xmlsec.tree.find_parent(root, consts.NodeSignedInfo)) @@ -27,7 +27,7 @@ def test_find_parent_bad_args(self): xmlsec.tree.find_parent('', 0, True) def test_find_node(self): - root = self.load_xml("sign_template.xml") + root = self.load_xml('sign_template.xml') ref = xmlsec.tree.find_node(root, consts.NodeReference) self.assertEqual(consts.NodeReference, ref.tag.partition('}')[2]) self.assertIsNone(xmlsec.tree.find_node(root, consts.NodeReference, consts.EncNs)) @@ -37,8 +37,8 @@ def test_find_node_bad_args(self): xmlsec.tree.find_node('', 0, True) def test_add_ids(self): - root = self.load_xml("sign_template.xml") - xmlsec.tree.add_ids(root, ["id1", "id2", "id3"]) + root = self.load_xml('sign_template.xml') + xmlsec.tree.add_ids(root, ['id1', 'id2', 'id3']) def test_add_ids_bad_args(self): with self.assertRaises(TypeError): diff --git a/tests/test_type_stubs.py b/tests/test_type_stubs.py index 9ed8f1e2..82f7df7f 100644 --- a/tests/test_type_stubs.py +++ b/tests/test_type_stubs.py @@ -40,8 +40,7 @@ class __TransformNoHref(NamedTuple): # __Transform type def gen_constants_stub(): - """ - Generate contents of the file:`xmlsec/constants.pyi`. + """Generate contents of the file:`xmlsec/constants.pyi`. Simply load all constants at runtime, generate appropriate type hint for each constant type. @@ -53,7 +52,7 @@ def process_constant(name): type_name = type(obj).__name__ if type_name in ('__KeyData', '__Transform') and obj.href is None: type_name += 'NoHref' - return '{name}: Final[{type_name}]'.format(name=name, type_name=type_name) + return f'{name}: Final[{type_name}]' names = list(sorted(name for name in dir(xmlsec.constants) if not name.startswith('__'))) lines = [process_constant(name) for name in names] @@ -61,8 +60,7 @@ def process_constant(name): def test_xmlsec_constants_stub(request): - """ - Generate the stub file for :mod:`xmlsec.constants` from existing code. + """Generate the stub file for :mod:`xmlsec.constants` from existing code. Compare it against the existing stub :file:`xmlsec/constants.pyi`. """ diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 303d7f8f..52dce2b3 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -4,8 +4,7 @@ class TestModule(base.TestMemoryLeaks): def test_reinitialize_module(self): - """ - This test doesn't explicitly verify anything, but will be invoked first in the suite. + """This test doesn't explicitly verify anything, but will be invoked first in the suite. So if the subsequent tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. From a89f8043c1b54e82b0ae38fcbfa237a6358e0dac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 21:19:19 +0200 Subject: [PATCH 186/211] [pre-commit.ci] pre-commit autoupdate (#360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.3 → v0.12.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.3...v0.12.4) - [github.com/pre-commit/mirrors-mypy: v1.16.1 → v1.17.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.16.1...v1.17.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48abcc6f..315bd1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.3 + rev: v0.12.4 hooks: - id: ruff args: ["--fix"] @@ -28,7 +28,7 @@ repos: args: [--autofix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.1 + rev: v1.17.0 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From b2e4bd6be9b509a5f656a544199e8ca88b65106b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:07:58 +0200 Subject: [PATCH 187/211] [pre-commit.ci] pre-commit autoupdate (#363) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.4 → v0.12.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.4...v0.12.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 315bd1ad..0f038b24 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.4 + rev: v0.12.5 hooks: - id: ruff args: ["--fix"] From adb329ee06b08f270209687650531f70a121ff1f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:34:47 +0200 Subject: [PATCH 188/211] [pre-commit.ci] pre-commit autoupdate (#365) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f038b24..0e6d0afc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.5 + rev: v0.12.7 hooks: - id: ruff args: ["--fix"] @@ -28,7 +28,7 @@ repos: args: [--autofix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.17.0 + rev: v1.17.1 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From b70c0db61988c70c196dbc6bd719be4b92a2b4d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:15:09 +0200 Subject: [PATCH 189/211] [pre-commit.ci] pre-commit autoupdate (#370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.7 → v0.12.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.7...v0.12.8) - [github.com/pre-commit/pre-commit-hooks: v5.0.0 → v6.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v5.0.0...v6.0.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e6d0afc..14395966 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.7 + rev: v0.12.8 hooks: - id: ruff args: ["--fix"] @@ -11,7 +11,7 @@ repos: types: [python] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace From 511d97b4fe99d81ab9cfdeff8405f1413d2f755c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:34:56 +0200 Subject: [PATCH 190/211] [pre-commit.ci] pre-commit autoupdate (#372) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.8 → v0.12.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.8...v0.12.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14395966..ca70eb16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.8 + rev: v0.12.9 hooks: - id: ruff args: ["--fix"] From d7cc117147711eb33464cf35eddce1b4bb65cc4d Mon Sep 17 00:00:00 2001 From: Paul Zakin Date: Wed, 20 Aug 2025 04:25:25 -0400 Subject: [PATCH 191/211] Update Alpine linux installation document (#373) --- README.md | 2 +- doc/source/install.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6abd6a5a..60bde880 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ port install libxml2 xmlsec pkgconfig ### Alpine ``` bash -apk add build-base libressl libffi-dev libressl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec +apk add build-base openssl libffi-dev openssl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec ``` ## Troubleshooting diff --git a/doc/source/install.rst b/doc/source/install.rst index 834b9acb..c892a3ea 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -59,7 +59,7 @@ Alpine .. code-block:: bash - apk add build-base libressl libffi-dev libressl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec + apk add build-base openssl libffi-dev openssl-dev libxslt-dev libxml2-dev xmlsec-dev xmlsec Troubleshooting From 71ad25b101556de2824f98b9a63350a7c7568c88 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:07:18 +0200 Subject: [PATCH 192/211] [pre-commit.ci] pre-commit autoupdate (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.9 → v0.12.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.9...v0.12.10) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca70eb16..deef08b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.9 + rev: v0.12.10 hooks: - id: ruff args: ["--fix"] From 5f82f6b4e7b9468ebc4fb7225fc6dcbaadcb3dce Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 1 Sep 2025 14:28:51 +0200 Subject: [PATCH 193/211] Bump libxml2 version to 2.14.5 for building wheels (#375) The goal is to keep it sync with lxml builds which in version 6.0.1 is using libxml2-2.14.5. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 62d48b4b..f29019f0 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -110,7 +110,7 @@ jobs: include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} env: - PYXMLSEC_LIBXML2_VERSION: 2.14.4 + PYXMLSEC_LIBXML2_VERSION: 2.14.5 PYXMLSEC_LIBXSLT_VERSION: 1.1.43 steps: From 3933936ece71dc82bfdfa4ab35d80869e7af2f57 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 14:39:28 +0200 Subject: [PATCH 194/211] [pre-commit.ci] pre-commit autoupdate (#376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.10 → v0.12.11](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.10...v0.12.11) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index deef08b2..46ee7c85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.10 + rev: v0.12.11 hooks: - id: ruff args: ["--fix"] From 699aa1e63c883adaa54c2e795446c529ec516ca7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:56:47 +0200 Subject: [PATCH 195/211] [pre-commit.ci] pre-commit autoupdate (#377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.11 → v0.12.12](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.11...v0.12.12) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46ee7c85..11bf9cc6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.11 + rev: v0.12.12 hooks: - id: ruff args: ["--fix"] From 5216e038615fc02466a51427a7f9f132e96bd884 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 12 Sep 2025 09:29:54 +0200 Subject: [PATCH 196/211] Add missing typing for id and ns parameters of template.create function (#378) --- src/xmlsec/template.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xmlsec/template.pyi b/src/xmlsec/template.pyi index a5199fc8..d1755fa2 100644 --- a/src/xmlsec/template.pyi +++ b/src/xmlsec/template.pyi @@ -15,7 +15,7 @@ def add_reference( ) -> _Element: ... def add_transform(node: _Element, transform: Transform) -> Any: ... def add_x509_data(node: _Element) -> _Element: ... -def create(node: _Element, c14n_method: Transform, sign_method: Transform) -> _Element: ... +def create(node: _Element, c14n_method: Transform, sign_method: Transform, id: str | None = ..., ns: str | None = ...) -> _Element: ... def encrypted_data_create( node: _Element, method: Transform, From 869e853d2396e1456bfc079d321ad38adadc71cd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Sep 2025 08:31:09 +0200 Subject: [PATCH 197/211] [pre-commit.ci] pre-commit autoupdate (#379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.12 → v0.13.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.12...v0.13.0) - [github.com/pre-commit/mirrors-mypy: v1.17.1 → v1.18.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.17.1...v1.18.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11bf9cc6..6403b4cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.12 + rev: v0.13.0 hooks: - id: ruff args: ["--fix"] @@ -28,7 +28,7 @@ repos: args: [--autofix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.17.1 + rev: v1.18.1 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From 2075d146e849847257893958eb5a30cd1943f0b1 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 22 Sep 2025 21:09:30 +0200 Subject: [PATCH 198/211] Lock libxml2 to v2.14.6 in manylinux workflow (#380) It seems that xmlsec library build fails with libxml2 v2.15.0. Let's lock it for now until the issue is resolved in the main repository. --- .github/workflows/manylinux.yml | 1 + tests/softhsm_setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 357e93eb..550a8b34 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -32,6 +32,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true + PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 until the issue with 2.15.x is resolved; e.g. https://github.com/lsh123/xmlsec/issues/948 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 3f5076d2..d55c16fd 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -140,7 +140,7 @@ def setup() -> None: ) logging.debug('Initializing the token') - out, err = run_cmd( + _, _ = run_cmd( [ component_path['SOFTHSM'], '--slot', From 6786271f4b1d580194dd6f6121e0f4f0fbb5c3df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 21:09:50 +0200 Subject: [PATCH 199/211] [pre-commit.ci] pre-commit autoupdate (#381) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.13.0 → v0.13.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.13.0...v0.13.1) - [github.com/pre-commit/mirrors-mypy: v1.18.1 → v1.18.2](https://github.com/pre-commit/mirrors-mypy/compare/v1.18.1...v1.18.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6403b4cc..5e38393e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.0 + rev: v0.13.1 hooks: - id: ruff args: ["--fix"] @@ -28,7 +28,7 @@ repos: args: [--autofix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.18.1 + rev: v1.18.2 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) From e24a84c1503e5ca250f163a71bc366551d370604 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 11:16:38 +0200 Subject: [PATCH 200/211] [pre-commit.ci] pre-commit autoupdate (#382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.13.1 → v0.13.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.13.1...v0.13.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e38393e..2a20f7cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.1 + rev: v0.13.2 hooks: - id: ruff args: ["--fix"] From 11beda8c3fddf680c8d0e315bb20cbf4d2edfd0e Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 6 Oct 2025 21:13:49 +0200 Subject: [PATCH 201/211] Replace urlretrieve with streaming download using urlopen (#384) Updated setup.py to replace urllib.request.urlretrieve with a streaming download helper based on urlopen. This avoids the drawbacks of urlretrieve, such as lack of streaming support and limited control over requests. Using urlopen allows downloads to be written in chunks, reduces memory usage, and makes it easier to extend the logic with custom headers, retries, or checksum verification in the future. This improves the reliability and security of fetching dependency tarballs during the build process. --- .github/workflows/manylinux.yml | 2 +- setup.py | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 550a8b34..04645e84 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -32,7 +32,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true - PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 until the issue with 2.15.x is resolved; e.g. https://github.com/lsh123/xmlsec/issues/948 + PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 to match it with lxml GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build diff --git a/setup.py b/setup.py index 94b49aa8..014476ca 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from distutils.version import StrictVersion as Version from pathlib import Path from urllib.parse import urljoin -from urllib.request import Request, urlcleanup, urlopen, urlretrieve +from urllib.request import Request, urlcleanup, urlopen from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig @@ -106,6 +106,16 @@ def latest_xmlsec_release(): return tar_gz['browser_download_url'] +def download_lib(url, filename): + req = Request(url, headers={'User-Agent': 'python-xmlsec build'}) + with urlopen(req) as r, open(filename, 'wb') as f: + while True: + chunk = r.read(8192) + if not chunk: + break + f.write(chunk) + + class CrossCompileInfo: def __init__(self, host, arch, compiler): self.host = host @@ -234,7 +244,7 @@ def prepare_static_build_win(self): else: self.info(f'Retrieving "{url}" to "{destfile}"') urlcleanup() # work around FTP bug 27973 in Py2.7.12+ - urlretrieve(url, str(destfile)) + download_lib(url, str(destfile)) for p in self.libs_dir.glob('*.zip'): with zipfile.ZipFile(str(p)) as f: @@ -297,7 +307,7 @@ def prepare_static_build(self, build_platform): else: url = f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{self.openssl_version}' self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - urlretrieve(url, str(openssl_tar)) + download_lib(url, str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) @@ -310,7 +320,7 @@ def prepare_static_build(self, build_platform): else: url = f'https://zlib.net/fossils/zlib-{self.zlib_version}.tar.gz' self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION={self.zlib_version}, downloading from {url}')) - urlretrieve(url, str(zlib_tar)) + download_lib(url, str(zlib_tar)) # fetch libiconv libiconv_tar = next(self.libs_dir.glob('libiconv*.tar.gz'), None) @@ -319,13 +329,13 @@ def prepare_static_build(self, build_platform): libiconv_tar = self.libs_dir / 'libiconv.tar.gz' if self.libiconv_version is None: url = latest_libiconv_release() - self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) + self.info('{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) else: url = f'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{self.libiconv_version}.tar.gz' self.info( - '{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') + '{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') ) - urlretrieve(url, str(libiconv_tar)) + download_lib(url, str(libiconv_tar)) # fetch libxml2 libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.xz'), None) @@ -341,7 +351,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION={self.libxml2_version}, downloading from {url}') ) libxml2_tar = self.libs_dir / 'libxml2.tar.xz' - urlretrieve(url, str(libxml2_tar)) + download_lib(url, str(libxml2_tar)) # fetch libxslt libxslt_tar = next(self.libs_dir.glob('libxslt*.tar.gz'), None) @@ -357,7 +367,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION={self.libxslt_version}, downloading from {url}') ) libxslt_tar = self.libs_dir / 'libxslt.tar.gz' - urlretrieve(url, str(libxslt_tar)) + download_lib(url, str(libxslt_tar)) # fetch xmlsec1 xmlsec1_tar = next(self.libs_dir.glob('xmlsec1*.tar.gz'), None) @@ -372,7 +382,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION={self.xmlsec1_version}, downloading from {url}') ) xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' - urlretrieve(url, str(xmlsec1_tar)) + download_lib(url, str(xmlsec1_tar)) for file in (openssl_tar, zlib_tar, libiconv_tar, libxml2_tar, libxslt_tar, xmlsec1_tar): self.info(f'Unpacking {file.name}') From 8d51c090a633cc52c178e7e89781696be4669d7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Oct 2025 07:47:10 +0200 Subject: [PATCH 202/211] [pre-commit.ci] pre-commit autoupdate (#385) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a20f7cb..bbb7e9eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.2 + rev: v0.13.3 hooks: - id: ruff args: ["--fix"] From e88d9ce1e2f7b757923c07c53658176c29b7ebb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:27:37 +0200 Subject: [PATCH 203/211] [pre-commit.ci] pre-commit autoupdate (#386) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.13.3 → v0.14.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.13.3...v0.14.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbb7e9eb..94fa772b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.3 + rev: v0.14.0 hooks: - id: ruff args: ["--fix"] From a241c56750f40cacd90090a58ed8277dc5f3254a Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 16 Oct 2025 19:39:06 +0800 Subject: [PATCH 204/211] build riscv64 wheel (#387) --- .github/workflows/wheels.yml | 4 ++-- doc/source/requirements.txt | 2 +- pyproject.toml | 5 +++-- requirements-test.txt | 2 +- requirements.txt | 2 +- setup.py | 16 ++++++++++++++-- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f29019f0..d9cad9ce 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -77,7 +77,7 @@ jobs: - uses: actions/checkout@v4 - name: Install cibuildwheel # Nb. keep cibuildwheel version pin consistent with job below - run: pipx install cibuildwheel==2.21.3 + run: pipx install cibuildwheel==3.1.4 - id: set-matrix # Once we have the windows build figured out, it can be added here # by updating the matrix to include windows builds as well. @@ -126,7 +126,7 @@ jobs: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.21.3 + uses: pypa/cibuildwheel@v3.1.4 with: only: ${{ matrix.only }} env: diff --git a/doc/source/requirements.txt b/doc/source/requirements.txt index 6b78694a..ffb2b6d3 100644 --- a/doc/source/requirements.txt +++ b/doc/source/requirements.txt @@ -1,4 +1,4 @@ -lxml>=3.8 +lxml==6.0.2 importlib_metadata;python_version < '3.8' packaging Sphinx>=3 diff --git a/pyproject.toml b/pyproject.toml index bb0c459e..2b2d2d04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4", "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] +requires = ["setuptools==80.9.0", "wheel", "setuptools_scm[toml]>=3.4", "pkgconfig>=1.5.1", "lxml==6.0.2"] [tool.mypy] files = ['src'] @@ -100,6 +100,7 @@ build-verbosity = 1 build-frontend = "build" skip = [ "pp*", # Skips PyPy builds (pp38-*, pp39-*, etc.) + "*musllinux_riscv64" # maturin and ruff currently don’t support the musl + riscv64 target ] test-command = "pytest -v --color=yes {package}/tests" before-test = "pip install -r requirements-test.txt" @@ -109,7 +110,7 @@ test-skip = "*-macosx_arm64" PYXMLSEC_STATIC_DEPS = "true" [tool.cibuildwheel.linux] -archs = ["x86_64", "aarch64"] +archs = ["x86_64", "aarch64", "riscv64"] environment-pass = [ "PYXMLSEC_LIBXML2_VERSION", "PYXMLSEC_LIBXSLT_VERSION", diff --git a/requirements-test.txt b/requirements-test.txt index 52a31f00..ad135d97 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,4 +2,4 @@ pytest==8.4.1 lxml-stubs==0.5.1 -ruff[format]==0.12.3 +ruff[format]==0.13.0 diff --git a/requirements.txt b/requirements.txt index 7aa6718b..8221c374 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -lxml==6.0.0 +lxml==6.0.2 diff --git a/setup.py b/setup.py index 014476ca..624169c4 100644 --- a/setup.py +++ b/setup.py @@ -404,9 +404,8 @@ def prepare_static_build(self, build_platform): ldflags.append(env['LDFLAGS']) cross_compiling = False - if build_platform == 'darwin': - import platform + if build_platform == 'darwin': arch = self.plat_name.rsplit('-', 1)[1] if arch != platform.machine() and arch in ('x86_64', 'arm64'): self.info(f'Cross-compiling for {arch}') @@ -423,6 +422,19 @@ def prepare_static_build(self, build_platform): self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if platform.machine() == 'riscv64': + # add `no-asm` flag for openssl while build for riscv64 + + # on openssl 3.5.2, When building for riscv64, ASM code is used by default. + # However, this version of the assembly code will report during the build: + + # relocation truncated to fit: R_RISCV_JAL against symbol `AES_set_encrypt_key' defined in .text section in /project/build/tmp/prefix/lib/libcrypto.a(libcrypto-lib-aes-riscv64.o) # noqa: E501 + + # This [line] (https://github.com/openssl/openssl/blob/0893a62353583343eb712adef6debdfbe597c227/crypto/aes/asm/aes-riscv64.pl#L1069) of code cannot be completed normally because the jump address of the static link library used by xmlsec is too large. # noqa: E501 + # may be we can consider filing a related issue with OpenSSL later. + # However, for now, for convenience, we can simply disable ASM for riscv64. + # This will result in some performance degradation, but this is acceptable. + openssl_config_cmd.append('no-asm') if cross_compiling: openssl_config_cmd.insert(0, './Configure') openssl_config_cmd.append(cross_compiling.triplet) From c8b1b6b611f0f7d3f710031f2a1597eb8c2ad4fa Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 17 Oct 2025 09:28:43 +0200 Subject: [PATCH 205/211] Refactor packaging build helpers into dedicated modules (#388) Split the old monolithic `setup.py` build logic into focused helper modules so that static builds are coordinated by dedicated utilities rather than one giant script. Additional changes: - Use fixed library versions - Use GNU mirror FTP for downloading related libraries --- .github/workflows/wheels.yml | 2 +- .pre-commit-config.yaml | 2 +- build_support/__init__.py | 0 build_support/build_ext.py | 86 +++++ build_support/network.py | 29 ++ build_support/releases.py | 77 +++++ build_support/static_build.py | 453 ++++++++++++++++++++++++++ setup.py | 587 +--------------------------------- 8 files changed, 651 insertions(+), 585 deletions(-) create mode 100644 build_support/__init__.py create mode 100644 build_support/build_ext.py create mode 100644 build_support/network.py create mode 100644 build_support/releases.py create mode 100644 build_support/static_build.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d9cad9ce..1d4564a6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -110,7 +110,7 @@ jobs: include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} env: - PYXMLSEC_LIBXML2_VERSION: 2.14.5 + PYXMLSEC_LIBXML2_VERSION: 2.14.6 PYXMLSEC_LIBXSLT_VERSION: 1.1.43 steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94fa772b..8fb6f59f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: rev: v1.18.2 hooks: - id: mypy - exclude: (setup.py|tests/.*.py|doc/.*) + exclude: (setup.py|tests|build_support/.*.py|doc/.*) types: [] files: ^.*.pyi?$ additional_dependencies: [lxml-stubs, types-docutils] diff --git a/build_support/__init__.py b/build_support/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/build_support/build_ext.py b/build_support/build_ext.py new file mode 100644 index 00000000..c4fb5bc9 --- /dev/null +++ b/build_support/build_ext.py @@ -0,0 +1,86 @@ +import os +import sys +from distutils import log +from distutils.errors import DistutilsError + +from setuptools.command.build_ext import build_ext as build_ext_orig + +from .static_build import CrossCompileInfo, StaticBuildHelper + + +class build_ext(build_ext_orig): + def info(self, message): + self.announce(message, level=log.INFO) + + def run(self): + ext = self.ext_map['xmlsec'] + self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) + self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) + self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) + + if self.static or sys.platform == 'win32': + helper = StaticBuildHelper(self) + helper.prepare(sys.platform) + else: + import pkgconfig + + try: + config = pkgconfig.parse('xmlsec1') + except OSError as error: + raise DistutilsError('Unable to invoke pkg-config.') from error + except pkgconfig.PackageNotFoundError as error: + raise DistutilsError('xmlsec1 is not installed or not in path.') from error + + if config is None or not config.get('libraries'): + raise DistutilsError('Bad or incomplete result returned from pkg-config.') + + ext.define_macros.extend(config['define_macros']) + ext.include_dirs.extend(config['include_dirs']) + ext.library_dirs.extend(config['library_dirs']) + ext.libraries.extend(config['libraries']) + + import lxml + + ext.include_dirs.extend(lxml.get_include()) + + ext.define_macros.extend( + [('MODULE_NAME', self.distribution.metadata.name), ('MODULE_VERSION', self.distribution.metadata.version)] + ) + for key, value in ext.define_macros: + if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): + ext.define_macros.remove((key, value)) + ext.define_macros.append((key, f'"{value}"')) + break + + if sys.platform == 'win32': + ext.extra_compile_args.append('/Zi') + else: + ext.extra_compile_args.extend( + [ + '-g', + '-std=c99', + '-fPIC', + '-fno-strict-aliasing', + '-Wno-error=declaration-after-statement', + '-Werror=implicit-function-declaration', + ] + ) + + if self.debug: + ext.define_macros.append(('PYXMLSEC_ENABLE_DEBUG', '1')) + if sys.platform == 'win32': + ext.extra_compile_args.append('/Od') + else: + ext.extra_compile_args.append('-Wall') + ext.extra_compile_args.append('-O0') + else: + if self.size_opt: + if sys.platform == 'win32': + ext.extra_compile_args.append('/Os') + else: + ext.extra_compile_args.append('-Os') + + super().run() + + +__all__ = ('CrossCompileInfo', 'build_ext') diff --git a/build_support/network.py b/build_support/network.py new file mode 100644 index 00000000..7ac0bb5e --- /dev/null +++ b/build_support/network.py @@ -0,0 +1,29 @@ +import contextlib +import json +from urllib.request import Request, urlopen + +DEFAULT_USER_AGENT = 'https://github.com/xmlsec/python-xmlsec' +DOWNLOAD_USER_AGENT = 'python-xmlsec build' + + +def make_request(url, github_token=None, json_response=False): + headers = {'User-Agent': DEFAULT_USER_AGENT} + if github_token: + headers['authorization'] = 'Bearer ' + github_token + request = Request(url, headers=headers) + with contextlib.closing(urlopen(request)) as response: + charset = response.headers.get_content_charset() or 'utf-8' + content = response.read().decode(charset) + if json_response: + return json.loads(content) + return content + + +def download_lib(url, filename): + request = Request(url, headers={'User-Agent': DOWNLOAD_USER_AGENT}) + with urlopen(request) as response, open(filename, 'wb') as target: + while True: + chunk = response.read(8192) + if not chunk: + break + target.write(chunk) diff --git a/build_support/releases.py b/build_support/releases.py new file mode 100644 index 00000000..089162e2 --- /dev/null +++ b/build_support/releases.py @@ -0,0 +1,77 @@ +import html.parser +import os +import re +from distutils import log +from distutils.version import StrictVersion as Version + +from .network import make_request + + +class HrefCollector(html.parser.HTMLParser): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for name, value in attrs: + if name == 'href': + self.hrefs.append(value) + + +def latest_release_from_html(url, matcher): + content = make_request(url) + collector = HrefCollector() + collector.feed(content) + hrefs = collector.hrefs + + def comp(text): + try: + return Version(matcher.match(text).groupdict()['version']) + except (AttributeError, ValueError): + return Version('0.0') + + latest = max(hrefs, key=comp) + return f'{url}/{latest}' + + +def latest_release_from_gnome_org_cache(url, lib_name): + cache_url = f'{url}/cache.json' + cache = make_request(cache_url, json_response=True) + latest_version = cache[2][lib_name][-1] + latest_source = cache[1][lib_name][latest_version]['tar.xz'] + return f'{url}/{latest_source}' + + +def latest_release_json_from_github_api(repo): + api_url = f'https://api.github.com/repos/{repo}/releases/latest' + token = os.environ.get('GH_TOKEN') + if token: + log.info('Using GitHub token to avoid rate limiting') + return make_request(api_url, token, json_response=True) + + +def latest_openssl_release(): + return latest_release_json_from_github_api('openssl/openssl')['tarball_url'] + + +def latest_zlib_release(): + return latest_release_from_html('https://zlib.net/fossils', re.compile('zlib-(?P.*).tar.gz')) + + +def latest_libiconv_release(): + return latest_release_from_html('https://ftpmirror.gnu.org/libiconv', re.compile('libiconv-(?P.*).tar.gz')) + + +def latest_libxml2_release(): + return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxml2', 'libxml2') + + +def latest_libxslt_release(): + return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxslt', 'libxslt') + + +def latest_xmlsec_release(): + assets = latest_release_json_from_github_api('lsh123/xmlsec')['assets'] + (tar_gz,) = [asset for asset in assets if asset['name'].endswith('.tar.gz')] + return tar_gz['browser_download_url'] diff --git a/build_support/static_build.py b/build_support/static_build.py new file mode 100644 index 00000000..4a0b98e8 --- /dev/null +++ b/build_support/static_build.py @@ -0,0 +1,453 @@ +import multiprocessing +import os +import platform +import subprocess +import sys +import tarfile +import zipfile +from distutils.errors import DistutilsError +from pathlib import Path +from urllib.parse import urljoin +from urllib.request import urlcleanup + +from .network import download_lib +from .releases import ( + latest_libiconv_release, + latest_libxml2_release, + latest_libxslt_release, + latest_openssl_release, + latest_xmlsec_release, + latest_zlib_release, +) + + +class CrossCompileInfo: + def __init__(self, host, arch, compiler): + self.host = host + self.arch = arch + self.compiler = compiler + + @property + def triplet(self): + return f'{self.host}-{self.arch}-{self.compiler}' + + +class StaticBuildHelper: + def __init__(self, builder): + self.builder = builder + self.ext = builder.ext_map['xmlsec'] + self.info = builder.info + self._prepare_directories() + + def prepare(self, platform_name): + self.info(f'starting static build on {sys.platform}') + if platform_name == 'win32': + self._prepare_windows_build() + elif 'linux' in platform_name or 'darwin' in platform_name: + self._prepare_unix_build(platform_name) + else: + raise DistutilsError(f'Unsupported static build platform: {platform_name}') + + def _prepare_directories(self): + buildroot = Path('build', 'tmp') + + prefix_dir = buildroot / 'prefix' + prefix_dir.mkdir(parents=True, exist_ok=True) + self.prefix_dir = prefix_dir.absolute() + + build_libs_dir = buildroot / 'libs' + build_libs_dir.mkdir(exist_ok=True) + self.build_libs_dir = build_libs_dir + + libs_dir = Path(os.environ.get('PYXMLSEC_LIBS_DIR', 'libs')) + libs_dir.mkdir(exist_ok=True) + self.libs_dir = libs_dir + self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) + + self.builder.prefix_dir = self.prefix_dir + self.builder.build_libs_dir = self.build_libs_dir + self.builder.libs_dir = self.libs_dir + + def _prepare_windows_build(self): + release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' + if platform.machine() == 'ARM64': + suffix = 'win-arm64' + elif sys.maxsize > 2**32: + suffix = 'win64' + else: + suffix = 'win32' + + libs = [ + f'libxml2-2.11.9-3.{suffix}.zip', + f'libxslt-1.1.39.{suffix}.zip', + f'zlib-1.3.1.{suffix}.zip', + f'iconv-1.18-1.{suffix}.zip', + f'openssl-3.0.16.pl1.{suffix}.zip', + f'xmlsec-1.3.7.{suffix}.zip', + ] + + for libfile in libs: + url = urljoin(release_url, libfile) + destfile = self.libs_dir / libfile + if destfile.is_file(): + self.info(f'Using local copy of "{url}"') + else: + self.info(f'Retrieving "{url}" to "{destfile}"') + urlcleanup() + download_lib(url, str(destfile)) + + for package in self.libs_dir.glob('*.zip'): + with zipfile.ZipFile(str(package)) as archive: + destdir = self.build_libs_dir + archive.extractall(path=str(destdir)) + + self.ext.define_macros = [ + ('XMLSEC_CRYPTO', '\\"openssl\\"'), + ('__XMLSEC_FUNCTION__', '__FUNCTION__'), + ('XMLSEC_NO_GOST', '1'), + ('XMLSEC_NO_XKMS', '1'), + ('XMLSEC_NO_CRYPTO_DYNAMIC_LOADING', '1'), + ('XMLSEC_CRYPTO_OPENSSL', '1'), + ('UNICODE', '1'), + ('_UNICODE', '1'), + ('LIBXML_ICONV_ENABLED', 1), + ('LIBXML_STATIC', '1'), + ('LIBXSLT_STATIC', 1), + ('XMLSEC_STATIC', 1), + ('inline', '__inline'), + ] + self.ext.libraries = [ + 'libxmlsec_a', + 'libxmlsec-openssl_a', + 'libcrypto', + 'iconv_a', + 'libxslt_a', + 'libexslt_a', + 'libxml2_a', + 'zlib', + 'WS2_32', + 'Advapi32', + 'User32', + 'Gdi32', + 'Crypt32', + ] + self.ext.library_dirs = [str(path.absolute()) for path in self.build_libs_dir.rglob('lib')] + + includes = [path for path in self.build_libs_dir.rglob('include') if path.is_dir()] + includes.append(next(path / 'xmlsec' for path in includes if (path / 'xmlsec').is_dir())) + self.ext.include_dirs = [str(path.absolute()) for path in includes] + + def _prepare_unix_build(self, build_platform): + self._capture_version_overrides() + archives = self._ensure_source_archives() + self._extract_archives(archives) + + env, prefix_arg, ldflags, cross_compile = self._prepare_build_environment(build_platform) + self._build_dependencies(env, prefix_arg, ldflags, cross_compile) + self._configure_extension_for_static(build_platform) + + def _capture_version_overrides(self): + builder = self.builder + builder.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '3.6.0') + builder.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION', '1.18') + builder.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', '2.14.6') + builder.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION', '1.1.43') + builder.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.3.1') + builder.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.3.8') + + def _ensure_source_archives(self): + return [ + self._ensure_source( + name='OpenSSL', + glob='openssl*.tar.gz', + filename='openssl.tar.gz', + version=self.builder.openssl_version, + env_label='PYXMLSEC_OPENSSL_VERSION', + default_url=latest_openssl_release, + version_url=lambda v: f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{v}', + ), + self._ensure_source( + name='zlib', + glob='zlib*.tar.gz', + filename='zlib.tar.gz', + version=self.builder.zlib_version, + env_label='PYXMLSEC_ZLIB_VERSION', + default_url=latest_zlib_release, + version_url=lambda v: f'https://zlib.net/fossils/zlib-{v}.tar.gz', + ), + self._ensure_source( + name='libiconv', + glob='libiconv*.tar.gz', + filename='libiconv.tar.gz', + version=self.builder.libiconv_version, + env_label='PYXMLSEC_LIBICONV_VERSION', + default_url=latest_libiconv_release, + version_url=lambda v: f'https://ftpmirror.gnu.org/libiconv/libiconv-{v}.tar.gz', + ), + self._ensure_source( + name='libxml2', + glob='libxml2*.tar.xz', + filename='libxml2.tar.xz', + version=self.builder.libxml2_version, + env_label='PYXMLSEC_LIBXML2_VERSION', + default_url=latest_libxml2_release, + version_url=lambda v: self._libxml_related_url('libxml2', v), + ), + self._ensure_source( + name='libxslt', + glob='libxslt*.tar.xz', + filename='libxslt.tar.xz', + version=self.builder.libxslt_version, + env_label='PYXMLSEC_LIBXSLT_VERSION', + default_url=latest_libxslt_release, + version_url=lambda v: self._libxml_related_url('libxslt', v), + ), + self._ensure_source( + name='xmlsec1', + glob='xmlsec1*.tar.gz', + filename='xmlsec1.tar.gz', + version=self.builder.xmlsec1_version, + env_label='PYXMLSEC_XMLSEC1_VERSION', + default_url=latest_xmlsec_release, + version_url=lambda v: f'https://github.com/lsh123/xmlsec/releases/download/{v}/xmlsec1-{v}.tar.gz', + ), + ] + + def _ensure_source(self, name, glob, filename, version, env_label, default_url, version_url): + archive = next(self.libs_dir.glob(glob), None) + if archive is not None: + return archive + + self.info('{:10}: {}'.format(name, 'source tar not found, downloading ...')) + archive = self.libs_dir / filename + if version is None: + url = default_url() + self.info('{:10}: {}'.format(name, f'{env_label} unset, downloading latest from {url}')) + else: + url = version_url(version) + self.info('{:10}: {}'.format(name, f'{env_label}={version}, downloading from {url}')) + download_lib(url, str(archive)) + return archive + + def _libxml_related_url(self, lib_name, version): + version_prefix, _ = version.rsplit('.', 1) + return f'https://download.gnome.org/sources/{lib_name}/{version_prefix}/{lib_name}-{version}.tar.xz' + + def _extract_archives(self, archives): + for archive in archives: + self.info(f'Unpacking {archive.name}') + try: + with tarfile.open(str(archive)) as tar: + tar.extractall(path=str(self.build_libs_dir)) + except EOFError as error: + raise DistutilsError(f'Bad {archive.name} downloaded; remove it and try again.') from error + + def _prepare_build_environment(self, build_platform): + prefix_arg = f'--prefix={self.prefix_dir}' + env = os.environ.copy() + + cflags = [] + if env.get('CFLAGS'): + cflags.append(env['CFLAGS']) + cflags.append('-fPIC') + + ldflags = [] + if env.get('LDFLAGS'): + ldflags.append(env['LDFLAGS']) + + cross_compile = None + if build_platform == 'darwin': + arch = self.builder.plat_name.rsplit('-', 1)[1] + if arch != platform.machine() and arch in ('x86_64', 'arm64'): + self.info(f'Cross-compiling for {arch}') + cflags.append(f'-arch {arch}') + ldflags.append(f'-arch {arch}') + cross_compile = CrossCompileInfo('darwin64', arch, 'cc') + major_version, _ = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11 and 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' + + env['CFLAGS'] = ' '.join(cflags) + env['LDFLAGS'] = ' '.join(ldflags) + return env, prefix_arg, ldflags, cross_compile + + def _build_dependencies(self, env, prefix_arg, ldflags, cross_compile): + self._build_openssl(env, prefix_arg, cross_compile) + self._build_zlib(env, prefix_arg) + + host_arg = [f'--host={cross_compile.arch}'] if cross_compile else [] + self._build_libiconv(env, prefix_arg, host_arg) + self._build_libxml2(env, prefix_arg, host_arg) + self._build_libxslt(env, prefix_arg, host_arg) + + ldflags.append('-lpthread') + env['LDFLAGS'] = ' '.join(ldflags) + self._build_xmlsec1(env, prefix_arg, host_arg) + + def _build_openssl(self, env, prefix_arg, cross_compile): + self.info('Building OpenSSL') + openssl_dir = next(self.build_libs_dir.glob('openssl-*')) + openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if platform.machine() == 'riscv64': + # openssl(riscv64): disable ASM to avoid R_RISCV_JAL relocation failure on 3.5.2 + # OpenSSL 3.5.2 enables RISC-V64 AES assembly by default. When we statically + # link libcrypto alongside xmlsec, the AES asm path triggers a link-time error: + # relocation truncated to fit: R_RISCV_JAL against symbol `AES_set_encrypt_key' + # in .../libcrypto.a(libcrypto-lib-aes-riscv64.o) + # This appears to stem from a long-range jump emitted by the AES asm generator + # (see aes-riscv64.pl around L1069), which can exceed the JAL reach when objects + # end up far apart in the final static link. + # As a pragmatic workaround, disable ASM on riscv64 (pass `no-asm`) so the + # portable C implementation is used. This unblocks the build at the cost of + # some crypto performance on riscv64 only. + # Refs: + # - https://github.com/openssl/openssl/blob/0893a62/crypto/aes/asm/aes-riscv64.pl#L1069 + openssl_config_cmd.append('no-asm') + if cross_compile: + openssl_config_cmd.insert(0, './Configure') + openssl_config_cmd.append(cross_compile.triplet) + else: + openssl_config_cmd.insert(0, './config') + subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(openssl_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install_sw'], cwd=str(openssl_dir), env=env) + + def _build_zlib(self, env, prefix_arg): + self.info('Building zlib') + zlib_dir = next(self.build_libs_dir.glob('zlib-*')) + subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(zlib_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(zlib_dir), env=env) + + def _build_libiconv(self, env, prefix_arg, host_arg): + self.info('Building libiconv') + libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + *host_arg, + ], + cwd=str(libiconv_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libiconv_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libiconv_dir), env=env) + + def _build_libxml2(self, env, prefix_arg, host_arg): + self.info('Building LibXML2') + libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + '--without-lzma', + '--without-python', + f'--with-iconv={self.prefix_dir}', + f'--with-zlib={self.prefix_dir}', + *host_arg, + ], + cwd=str(libxml2_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxml2_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxml2_dir), env=env) + + def _build_libxslt(self, env, prefix_arg, host_arg): + self.info('Building libxslt') + libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + '--without-python', + '--without-crypto', + f'--with-libxml-prefix={self.prefix_dir}', + *host_arg, + ], + cwd=str(libxslt_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxslt_dir), env=env) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxslt_dir), env=env) + + def _build_xmlsec1(self, env, prefix_arg, host_arg): + self.info('Building xmlsec1') + xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) + subprocess.check_call( + [ + './configure', + prefix_arg, + '--disable-shared', + '--disable-gost', + '--enable-md5', + '--enable-ripemd160', + '--disable-crypto-dl', + '--enable-static=yes', + '--enable-shared=no', + '--enable-static-linking=yes', + '--with-default-crypto=openssl', + f'--with-openssl={self.prefix_dir}', + f'--with-libxml={self.prefix_dir}', + f'--with-libxslt={self.prefix_dir}', + *host_arg, + ], + cwd=str(xmlsec1_dir), + env=env, + ) + include_flags = [ + f'-I{self.prefix_dir / "include"}', + f'-I{self.prefix_dir / "include" / "libxml"}', + ] + subprocess.check_call( + ['make', f'-j{multiprocessing.cpu_count() + 1}', *include_flags], + cwd=str(xmlsec1_dir), + env=env, + ) + subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(xmlsec1_dir), env=env) + + def _configure_extension_for_static(self, build_platform): + self.ext.define_macros = [ + ('__XMLSEC_FUNCTION__', '__func__'), + ('XMLSEC_NO_SIZE_T', None), + ('XMLSEC_NO_GOST', '1'), + ('XMLSEC_NO_GOST2012', '1'), + ('XMLSEC_NO_XKMS', '1'), + ('XMLSEC_CRYPTO', '\\"openssl\\"'), + ('XMLSEC_NO_CRYPTO_DYNAMIC_LOADING', '1'), + ('XMLSEC_CRYPTO_OPENSSL', '1'), + ('LIBXML_ICONV_ENABLED', 1), + ('LIBXML_STATIC', 1), + ('LIBXSLT_STATIC', 1), + ('XMLSEC_STATIC', 1), + ('inline', '__inline'), + ('UNICODE', '1'), + ('_UNICODE', '1'), + ] + + self.ext.include_dirs.append(str(self.prefix_dir / 'include')) + self.ext.include_dirs.extend([str(path.absolute()) for path in (self.prefix_dir / 'include').iterdir() if path.is_dir()]) + + self.ext.library_dirs = [] + if build_platform == 'linux': + self.ext.libraries = ['m', 'rt'] + extra_objects = [ + 'libxmlsec1.a', + 'libxslt.a', + 'libxml2.a', + 'libz.a', + 'libxmlsec1-openssl.a', + 'libcrypto.a', + 'libiconv.a', + 'libxmlsec1.a', + ] + self.ext.extra_objects = [str(self.prefix_dir / 'lib' / obj) for obj in extra_objects] + + +__all__ = ('CrossCompileInfo', 'StaticBuildHelper') diff --git a/setup.py b/setup.py index 624169c4..4b3d93f7 100644 --- a/setup.py +++ b/setup.py @@ -1,596 +1,17 @@ -import contextlib -import html.parser -import json -import multiprocessing -import os -import platform -import re -import subprocess -import sys -import tarfile -import zipfile -from distutils import log -from distutils.errors import DistutilsError -from distutils.version import StrictVersion as Version from pathlib import Path -from urllib.parse import urljoin -from urllib.request import Request, urlcleanup, urlopen from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext as build_ext_orig - - -class HrefCollector(html.parser.HTMLParser): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.hrefs = [] - - def handle_starttag(self, tag, attrs): - if tag == 'a': - for name, value in attrs: - if name == 'href': - self.hrefs.append(value) - - -def make_request(url, github_token=None, json_response=False): - headers = {'User-Agent': 'https://github.com/xmlsec/python-xmlsec'} - if github_token: - headers['authorization'] = 'Bearer ' + github_token - request = Request(url, headers=headers) - with contextlib.closing(urlopen(request)) as r: - charset = r.headers.get_content_charset() or 'utf-8' - content = r.read().decode(charset) - if json_response: - return json.loads(content) - else: - return content - - -def latest_release_from_html(url, matcher): - content = make_request(url) - collector = HrefCollector() - collector.feed(content) - hrefs = collector.hrefs - - def comp(text): - try: - return Version(matcher.match(text).groupdict()['version']) - except (AttributeError, ValueError): - return Version('0.0') - - latest = max(hrefs, key=comp) - return f'{url}/{latest}' - - -def latest_release_from_gnome_org_cache(url, lib_name): - cache_url = f'{url}/cache.json' - cache = make_request(cache_url, json_response=True) - latest_version = cache[2][lib_name][-1] - latest_source = cache[1][lib_name][latest_version]['tar.xz'] - return f'{url}/{latest_source}' - - -def latest_release_json_from_github_api(repo): - api_url = f'https://api.github.com/repos/{repo}/releases/latest' - - # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited - token = os.environ.get('GH_TOKEN') - if token: - log.info('Using GitHub token to avoid rate limiting') - return make_request(api_url, token, json_response=True) - - -def latest_openssl_release(): - return latest_release_json_from_github_api('openssl/openssl')['tarball_url'] - - -def latest_zlib_release(): - return latest_release_from_html('https://zlib.net/fossils', re.compile('zlib-(?P.*).tar.gz')) - - -def latest_libiconv_release(): - return latest_release_from_html('https://ftp.gnu.org/pub/gnu/libiconv', re.compile('libiconv-(?P.*).tar.gz')) - - -def latest_libxml2_release(): - return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxml2', 'libxml2') - - -def latest_libxslt_release(): - return latest_release_from_gnome_org_cache('https://download.gnome.org/sources/libxslt', 'libxslt') - - -def latest_xmlsec_release(): - assets = latest_release_json_from_github_api('lsh123/xmlsec')['assets'] - (tar_gz,) = [asset for asset in assets if asset['name'].endswith('.tar.gz')] - return tar_gz['browser_download_url'] - - -def download_lib(url, filename): - req = Request(url, headers={'User-Agent': 'python-xmlsec build'}) - with urlopen(req) as r, open(filename, 'wb') as f: - while True: - chunk = r.read(8192) - if not chunk: - break - f.write(chunk) - - -class CrossCompileInfo: - def __init__(self, host, arch, compiler): - self.host = host - self.arch = arch - self.compiler = compiler - - @property - def triplet(self): - return f'{self.host}-{self.arch}-{self.compiler}' - - -class build_ext(build_ext_orig): - def info(self, message): - self.announce(message, level=log.INFO) - - def run(self): - ext = self.ext_map['xmlsec'] - self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) - self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) - self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) - - if self.static or sys.platform == 'win32': - self.info(f'starting static build on {sys.platform}') - buildroot = Path('build', 'tmp') - - self.prefix_dir = buildroot / 'prefix' - self.prefix_dir.mkdir(parents=True, exist_ok=True) - self.prefix_dir = self.prefix_dir.absolute() - - self.build_libs_dir = buildroot / 'libs' - self.build_libs_dir.mkdir(exist_ok=True) - - self.libs_dir = Path(os.environ.get('PYXMLSEC_LIBS_DIR', 'libs')) - self.libs_dir.mkdir(exist_ok=True) - self.info('{:20} {}'.format('Lib sources in:', self.libs_dir.absolute())) - - if sys.platform == 'win32': - self.prepare_static_build_win() - elif 'linux' in sys.platform or 'darwin' in sys.platform: - self.prepare_static_build(sys.platform) - else: - import pkgconfig - - try: - config = pkgconfig.parse('xmlsec1') - except OSError as e: - raise DistutilsError('Unable to invoke pkg-config.') from e - except pkgconfig.PackageNotFoundError as e: - raise DistutilsError('xmlsec1 is not installed or not in path.') from e - - if config is None or not config.get('libraries'): - raise DistutilsError('Bad or incomplete result returned from pkg-config.') - - ext.define_macros.extend(config['define_macros']) - ext.include_dirs.extend(config['include_dirs']) - ext.library_dirs.extend(config['library_dirs']) - ext.libraries.extend(config['libraries']) - - import lxml - - ext.include_dirs.extend(lxml.get_include()) - - ext.define_macros.extend( - [('MODULE_NAME', self.distribution.metadata.name), ('MODULE_VERSION', self.distribution.metadata.version)] - ) - # escape the XMLSEC_CRYPTO macro value, see mehcode/python-xmlsec#141 - for key, value in ext.define_macros: - if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): - ext.define_macros.remove((key, value)) - ext.define_macros.append((key, f'"{value}"')) - break - - if sys.platform == 'win32': - ext.extra_compile_args.append('/Zi') - else: - ext.extra_compile_args.extend( - [ - '-g', - '-std=c99', - '-fPIC', - '-fno-strict-aliasing', - '-Wno-error=declaration-after-statement', - '-Werror=implicit-function-declaration', - ] - ) - - if self.debug: - ext.define_macros.append(('PYXMLSEC_ENABLE_DEBUG', '1')) - if sys.platform == 'win32': - ext.extra_compile_args.append('/Od') - else: - ext.extra_compile_args.append('-Wall') - ext.extra_compile_args.append('-O0') - else: - if self.size_opt: - if sys.platform == 'win32': - ext.extra_compile_args.append('/Os') - else: - ext.extra_compile_args.append('-Os') - - super().run() - - def prepare_static_build_win(self): - release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2025.07.10/' - if platform.machine() == 'ARM64': - suffix = 'win-arm64' - elif sys.maxsize > 2**32: # 2.0 GiB - suffix = 'win64' - else: - suffix = 'win32' - - libs = [ - f'libxml2-2.11.9-3.{suffix}.zip', - f'libxslt-1.1.39.{suffix}.zip', - f'zlib-1.3.1.{suffix}.zip', - f'iconv-1.18-1.{suffix}.zip', - f'openssl-3.0.16.pl1.{suffix}.zip', - f'xmlsec-1.3.7.{suffix}.zip', - ] - - for libfile in libs: - url = urljoin(release_url, libfile) - destfile = self.libs_dir / libfile - if destfile.is_file(): - self.info(f'Using local copy of "{url}"') - else: - self.info(f'Retrieving "{url}" to "{destfile}"') - urlcleanup() # work around FTP bug 27973 in Py2.7.12+ - download_lib(url, str(destfile)) - - for p in self.libs_dir.glob('*.zip'): - with zipfile.ZipFile(str(p)) as f: - destdir = self.build_libs_dir - f.extractall(path=str(destdir)) - - ext = self.ext_map['xmlsec'] - ext.define_macros = [ - ('XMLSEC_CRYPTO', '\\"openssl\\"'), - ('__XMLSEC_FUNCTION__', '__FUNCTION__'), - ('XMLSEC_NO_GOST', '1'), - ('XMLSEC_NO_XKMS', '1'), - ('XMLSEC_NO_CRYPTO_DYNAMIC_LOADING', '1'), - ('XMLSEC_CRYPTO_OPENSSL', '1'), - ('UNICODE', '1'), - ('_UNICODE', '1'), - ('LIBXML_ICONV_ENABLED', 1), - ('LIBXML_STATIC', '1'), - ('LIBXSLT_STATIC', '1'), - ('XMLSEC_STATIC', '1'), - ('inline', '__inline'), - ] - ext.libraries = [ - 'libxmlsec_a', - 'libxmlsec-openssl_a', - 'libcrypto', - 'iconv_a', - 'libxslt_a', - 'libexslt_a', - 'libxml2_a', - 'zlib', - 'WS2_32', - 'Advapi32', - 'User32', - 'Gdi32', - 'Crypt32', - ] - ext.library_dirs = [str(p.absolute()) for p in self.build_libs_dir.rglob('lib')] - - includes = [p for p in self.build_libs_dir.rglob('include') if p.is_dir()] - includes.append(next(p / 'xmlsec' for p in includes if (p / 'xmlsec').is_dir())) - ext.include_dirs = [str(p.absolute()) for p in includes] - - def prepare_static_build(self, build_platform): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') - self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') - self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') - self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') - self.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION') - self.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION') - - # fetch openssl - openssl_tar = next(self.libs_dir.glob('openssl*.tar.gz'), None) - if openssl_tar is None: - self.info('{:10}: {}'.format('OpenSSL', 'source tar not found, downloading ...')) - openssl_tar = self.libs_dir / 'openssl.tar.gz' - if self.openssl_version is None: - url = latest_openssl_release() - self.info('{:10}: {}'.format('OpenSSL', f'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {url}')) - else: - url = f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{self.openssl_version}' - self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - download_lib(url, str(openssl_tar)) - - # fetch zlib - zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) - if zlib_tar is None: - self.info('{:10}: {}'.format('zlib', 'source not found, downloading ...')) - zlib_tar = self.libs_dir / 'zlib.tar.gz' - if self.zlib_version is None: - url = latest_zlib_release() - self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION unset, downloading latest from {url}')) - else: - url = f'https://zlib.net/fossils/zlib-{self.zlib_version}.tar.gz' - self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION={self.zlib_version}, downloading from {url}')) - download_lib(url, str(zlib_tar)) - - # fetch libiconv - libiconv_tar = next(self.libs_dir.glob('libiconv*.tar.gz'), None) - if libiconv_tar is None: - self.info('{:10}: {}'.format('libiconv', 'source not found, downloading ...')) - libiconv_tar = self.libs_dir / 'libiconv.tar.gz' - if self.libiconv_version is None: - url = latest_libiconv_release() - self.info('{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) - else: - url = f'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{self.libiconv_version}.tar.gz' - self.info( - '{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') - ) - download_lib(url, str(libiconv_tar)) - - # fetch libxml2 - libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.xz'), None) - if libxml2_tar is None: - self.info('{:10}: {}'.format('libxml2', 'source tar not found, downloading ...')) - if self.libxml2_version is None: - url = latest_libxml2_release() - self.info('{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION unset, downloading latest from {url}')) - else: - version_prefix, _ = self.libxml2_version.rsplit('.', 1) - url = f'https://download.gnome.org/sources/libxml2/{version_prefix}/libxml2-{self.libxml2_version}.tar.xz' - self.info( - '{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION={self.libxml2_version}, downloading from {url}') - ) - libxml2_tar = self.libs_dir / 'libxml2.tar.xz' - download_lib(url, str(libxml2_tar)) - - # fetch libxslt - libxslt_tar = next(self.libs_dir.glob('libxslt*.tar.gz'), None) - if libxslt_tar is None: - self.info('{:10}: {}'.format('libxslt', 'source tar not found, downloading ...')) - if self.libxslt_version is None: - url = latest_libxslt_release() - self.info('{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION unset, downloading latest from {url}')) - else: - version_prefix, _ = self.libxslt_version.rsplit('.', 1) - url = f'https://download.gnome.org/sources/libxslt/{version_prefix}/libxslt-{self.libxslt_version}.tar.xz' - self.info( - '{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION={self.libxslt_version}, downloading from {url}') - ) - libxslt_tar = self.libs_dir / 'libxslt.tar.gz' - download_lib(url, str(libxslt_tar)) - - # fetch xmlsec1 - xmlsec1_tar = next(self.libs_dir.glob('xmlsec1*.tar.gz'), None) - if xmlsec1_tar is None: - self.info('{:10}: {}'.format('xmlsec1', 'source tar not found, downloading ...')) - if self.xmlsec1_version is None: - url = latest_xmlsec_release() - self.info('{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION unset, downloading latest from {url}')) - else: - url = f'https://github.com/lsh123/xmlsec/releases/download/{self.xmlsec1_version}/xmlsec1-{self.xmlsec1_version}.tar.gz' - self.info( - '{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION={self.xmlsec1_version}, downloading from {url}') - ) - xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' - download_lib(url, str(xmlsec1_tar)) - - for file in (openssl_tar, zlib_tar, libiconv_tar, libxml2_tar, libxslt_tar, xmlsec1_tar): - self.info(f'Unpacking {file.name}') - try: - with tarfile.open(str(file)) as tar: - tar.extractall(path=str(self.build_libs_dir)) - except EOFError as e: - raise DistutilsError(f'Bad {file.name} downloaded; remove it and try again.') from e - - prefix_arg = f'--prefix={self.prefix_dir}' - - env = os.environ.copy() - cflags = [] - if env.get('CFLAGS'): - cflags.append(env['CFLAGS']) - cflags.append('-fPIC') - ldflags = [] - if env.get('LDFLAGS'): - ldflags.append(env['LDFLAGS']) - - cross_compiling = False - - if build_platform == 'darwin': - arch = self.plat_name.rsplit('-', 1)[1] - if arch != platform.machine() and arch in ('x86_64', 'arm64'): - self.info(f'Cross-compiling for {arch}') - cflags.append(f'-arch {arch}') - ldflags.append(f'-arch {arch}') - cross_compiling = CrossCompileInfo('darwin64', arch, 'cc') - major_version, _ = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) - if major_version >= 11 and 'MACOSX_DEPLOYMENT_TARGET' not in env: - env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' - - env['CFLAGS'] = ' '.join(cflags) - env['LDFLAGS'] = ' '.join(ldflags) - - self.info('Building OpenSSL') - openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] - if platform.machine() == 'riscv64': - # add `no-asm` flag for openssl while build for riscv64 - - # on openssl 3.5.2, When building for riscv64, ASM code is used by default. - # However, this version of the assembly code will report during the build: - - # relocation truncated to fit: R_RISCV_JAL against symbol `AES_set_encrypt_key' defined in .text section in /project/build/tmp/prefix/lib/libcrypto.a(libcrypto-lib-aes-riscv64.o) # noqa: E501 - - # This [line] (https://github.com/openssl/openssl/blob/0893a62353583343eb712adef6debdfbe597c227/crypto/aes/asm/aes-riscv64.pl#L1069) of code cannot be completed normally because the jump address of the static link library used by xmlsec is too large. # noqa: E501 - # may be we can consider filing a related issue with OpenSSL later. - # However, for now, for convenience, we can simply disable ASM for riscv64. - # This will result in some performance degradation, but this is acceptable. - openssl_config_cmd.append('no-asm') - if cross_compiling: - openssl_config_cmd.insert(0, './Configure') - openssl_config_cmd.append(cross_compiling.triplet) - else: - openssl_config_cmd.insert(0, './config') - subprocess.check_call(openssl_config_cmd, cwd=str(openssl_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(openssl_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install_sw'], cwd=str(openssl_dir), env=env) - - self.info('Building zlib') - zlib_dir = next(self.build_libs_dir.glob('zlib-*')) - subprocess.check_call(['./configure', prefix_arg], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(zlib_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(zlib_dir), env=env) - - host_arg = [] - if cross_compiling: - host_arg = [f'--host={cross_compiling.arch}'] - - self.info('Building libiconv') - libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - *host_arg, - ], - cwd=str(libiconv_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libiconv_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libiconv_dir), env=env) - - self.info('Building LibXML2') - libxml2_dir = next(self.build_libs_dir.glob('libxml2-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - '--without-lzma', - '--without-python', - f'--with-iconv={self.prefix_dir}', - f'--with-zlib={self.prefix_dir}', - *host_arg, - ], - cwd=str(libxml2_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxml2_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxml2_dir), env=env) - - self.info('Building libxslt') - libxslt_dir = next(self.build_libs_dir.glob('libxslt-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-dependency-tracking', - '--disable-shared', - '--without-python', - '--without-crypto', - f'--with-libxml-prefix={self.prefix_dir}', - *host_arg, - ], - cwd=str(libxslt_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}'], cwd=str(libxslt_dir), env=env) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(libxslt_dir), env=env) - - self.info('Building xmlsec1') - ldflags.append('-lpthread') - env['LDFLAGS'] = ' '.join(ldflags) - xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) - subprocess.check_call( - [ - './configure', - prefix_arg, - '--disable-shared', - '--disable-gost', - '--enable-md5', - '--enable-ripemd160', - '--disable-crypto-dl', - '--enable-static=yes', - '--enable-shared=no', - '--enable-static-linking=yes', - '--with-default-crypto=openssl', - f'--with-openssl={self.prefix_dir}', - f'--with-libxml={self.prefix_dir}', - f'--with-libxslt={self.prefix_dir}', - *host_arg, - ], - cwd=str(xmlsec1_dir), - env=env, - ) - include_flags = [ - f'-I{self.prefix_dir / "include"}', - f'-I{self.prefix_dir / "include" / "libxml"}', - ] - subprocess.check_call( - ['make', f'-j{multiprocessing.cpu_count() + 1}', *include_flags], - cwd=str(xmlsec1_dir), - env=env, - ) - subprocess.check_call(['make', f'-j{multiprocessing.cpu_count() + 1}', 'install'], cwd=str(xmlsec1_dir), env=env) - - ext = self.ext_map['xmlsec'] - ext.define_macros = [ - ('__XMLSEC_FUNCTION__', '__func__'), - ('XMLSEC_NO_SIZE_T', None), - ('XMLSEC_NO_GOST', '1'), - ('XMLSEC_NO_GOST2012', '1'), - ('XMLSEC_NO_XKMS', '1'), - ('XMLSEC_CRYPTO', '\\"openssl\\"'), - ('XMLSEC_NO_CRYPTO_DYNAMIC_LOADING', '1'), - ('XMLSEC_CRYPTO_OPENSSL', '1'), - ('LIBXML_ICONV_ENABLED', 1), - ('LIBXML_STATIC', '1'), - ('LIBXSLT_STATIC', '1'), - ('XMLSEC_STATIC', '1'), - ('inline', '__inline'), - ('UNICODE', '1'), - ('_UNICODE', '1'), - ] - - ext.include_dirs.append(str(self.prefix_dir / 'include')) - ext.include_dirs.extend([str(p.absolute()) for p in (self.prefix_dir / 'include').iterdir() if p.is_dir()]) - - ext.library_dirs = [] - if build_platform == 'linux': - ext.libraries = ['m', 'rt'] - extra_objects = [ - 'libxmlsec1.a', - 'libxslt.a', - 'libxml2.a', - 'libz.a', - 'libxmlsec1-openssl.a', - 'libcrypto.a', - 'libiconv.a', - 'libxmlsec1.a', - ] - ext.extra_objects = [str(self.prefix_dir / 'lib' / o) for o in extra_objects] +from build_support.build_ext import build_ext src_root = Path(__file__).parent / 'src' -sources = [str(p.absolute()) for p in src_root.rglob('*.c')] +sources = [str(path.absolute()) for path in src_root.rglob('*.c')] pyxmlsec = Extension('xmlsec', sources=sources) setup_reqs = ['setuptools_scm[toml]>=3.4', 'pkgconfig>=1.5.1', 'lxml>=3.8'] -with open('README.md', encoding='utf-8') as f: - long_desc = f.read() +with open('README.md', encoding='utf-8') as readme: + long_desc = readme.read() setup( From 47f3e00c2e351ad663a753e68c7c13b780627ef7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 12:12:56 +0200 Subject: [PATCH 206/211] [pre-commit.ci] pre-commit autoupdate (#389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.0 → v0.14.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.0...v0.14.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fb6f59f..3dc8df2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.0 + rev: v0.14.1 hooks: - id: ruff args: ["--fix"] From c42bf925d7eb7acdab0a892e326c7690fb009de1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:52:15 +0100 Subject: [PATCH 207/211] [pre-commit.ci] pre-commit autoupdate (#391) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.1 → v0.14.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.1...v0.14.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3dc8df2d..2a91f0ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.1 + rev: v0.14.2 hooks: - id: ruff args: ["--fix"] From 1bb232462fcb2b3b726a3b5443f30f34e34b9a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Escolano?= Date: Fri, 31 Oct 2025 15:54:13 +0100 Subject: [PATCH 208/211] Add support for Python 3.14 (#393) --- .github/workflows/linuxbrew.yml | 2 +- .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 2 +- .github/workflows/sdist.yml | 2 +- .travis.yml | 1 + pyproject.toml | 3 ++- setup.py | 1 + 7 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 8a231047..1c6d9543 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] env: # For some unknown reason, linuxbrew tries to use "gcc-11" by default, which doesn't exist. CC: gcc diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 44d214bc..e2e2a0df 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -8,7 +8,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] static_deps: ["static", ""] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 04645e84..d1c205d7 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313] + python-abi: [cp39-cp39, cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313, cp314-cp314] image: - manylinux2014_x86_64 - manylinux_2_28_x86_64 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 3bdc9764..ecc53c31 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 - name: Set up Python diff --git a/.travis.yml b/.travis.yml index 9574a1c7..8d3ca07e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ python: - "3.11" - "3.12" - "3.13" + - "3.14" env: global: diff --git a/pyproject.toml b/pyproject.toml index 2b2d2d04..7c7b4bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,8 @@ build = [ "cp310-*", "cp311-*", "cp312-*", - "cp313-*" + "cp313-*", + "cp314-*" ] build-verbosity = 1 build-frontend = "build" diff --git a/setup.py b/setup.py index 4b3d93f7..4100a52b 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From 60a0788e4359de80591a1c78bbf161325a7b1078 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Nov 2025 10:26:11 +0100 Subject: [PATCH 209/211] [pre-commit.ci] pre-commit autoupdate (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.2 → v0.14.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.2...v0.14.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a91f0ab..49a3a0da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.2 + rev: v0.14.3 hooks: - id: ruff args: ["--fix"] From 47d4201def5d9d131b4f4a29088d677fab8a19f4 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 10 Nov 2025 11:12:09 +0100 Subject: [PATCH 210/211] Bump xmlsec1 library to v1.3.9 (#398) --- build_support/static_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_support/static_build.py b/build_support/static_build.py index 4a0b98e8..09e2039a 100644 --- a/build_support/static_build.py +++ b/build_support/static_build.py @@ -153,7 +153,7 @@ def _capture_version_overrides(self): builder.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION', '2.14.6') builder.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION', '1.1.43') builder.zlib_version = os.environ.get('PYXMLSEC_ZLIB_VERSION', '1.3.1') - builder.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.3.8') + builder.xmlsec1_version = os.environ.get('PYXMLSEC_XMLSEC1_VERSION', '1.3.9') def _ensure_source_archives(self): return [ From 8d98ff24fbf03f530c3178e013ba42c8afc0d26f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Nov 2025 13:11:53 +0100 Subject: [PATCH 211/211] [pre-commit.ci] pre-commit autoupdate (#399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.3 → v0.14.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.3...v0.14.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49a3a0da..aca65390 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.3 + rev: v0.14.4 hooks: - id: ruff args: ["--fix"]