diff -Nru yara-python-4.3.1/.github/workflows/publish-to-pypi.yml yara-python-4.5.0/.github/workflows/publish-to-pypi.yml --- yara-python-4.3.1/.github/workflows/publish-to-pypi.yml 1970-01-01 00:00:00.000000000 +0000 +++ yara-python-4.5.0/.github/workflows/publish-to-pypi.yml 2024-02-14 09:10:08.000000000 +0000 @@ -0,0 +1,103 @@ +name: Publish Python distribution to PyPI + +on: + push: + branches: + - master + tags: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build wheels on ${{ matrix.os }} (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-22.04 + arch: x86_64 + - os: ubuntu-22.04 + arch: i686 + - os: ubuntu-22.04 + arch: aarch64 + - os: macos-12 + arch: x86_64 universal2 + - os: windows-2022 + arch: x86 + before: vcpkg install openssl:x86-windows-static + env: LIB="C:\\vcpkg\\packages\\openssl_x86-windows-static\\lib" INCLUDE="C:\\vcpkg\\packages\\openssl_x86-windows-static\\include" + - os: windows-2022 + arch: AMD64 + before: vcpkg install openssl:x64-windows-static + env: LIB="C:\\vcpkg\\packages\\openssl_x64-windows-static\\lib" INCLUDE="C:\\vcpkg\\packages\\openssl_x64-windows-static\\include" + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + with: + output-dir: dist + env: + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BEFORE_ALL_LINUX: | + if [[ ! -z "$(which yum)" ]]; then + yum install -y make gcc perl-core pcre-devel wget zlib-devel git automake + wget https://ftp.openssl.org/source/openssl-1.1.1k.tar.gz + tar xf openssl*.gz + cd openssl* + ./config --prefix=/usr --openssldir=/etc/ssl zlib-dynamic + make -j$(nproc) + make install + elif [[ ! -z "$(which apk)" ]]; then + apk add openssl-dev + fi + CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.before }} + CIBW_BUILD_FRONTEND: build + CIBW_CONFIG_SETTINGS: --enable-cuckoo --enable-magic --enable-dex --enable-macho --enable-openssl + CIBW_ENVIRONMENT: ${{ matrix.env }} + CIBW_SKIP: cp36-* + CIBW_TEST_COMMAND: python {package}/tests.py + + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions-${{ matrix.os }}-${{ matrix.arch }} + path: dist/*.whl + + publish-to-pypi: + needs: [build] + runs-on: ubuntu-latest + + # Only publish to PyPI on tag pushes + if: startsWith(github.ref, 'refs/tags/') + + environment: + name: pypi + url: https://pypi.org/p/yara-python + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + pattern: python-package-distributions-* + merge-multiple: true + path: dist/ + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff -Nru yara-python-4.3.1/appveyor.yml yara-python-4.5.0/appveyor.yml --- yara-python-4.3.1/appveyor.yml 2023-04-21 08:30:15.000000000 +0000 +++ yara-python-4.5.0/appveyor.yml 2024-02-14 09:10:08.000000000 +0000 @@ -7,22 +7,6 @@ JANSSON_VERSION: "2.13" matrix: - - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - # See: http://www.appveyor.com/docs/installed-software#python - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - OPENSSL_LIB: "https://ci.appveyor.com/api/buildjobs/fakubeldw67e9pmg/artifacts/YARA.OpenSSL.x86.1.1.1.nupkg" - VS: "Visual Studio 14 2015" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - OPENSSL_LIB: "https://ci.appveyor.com/api/buildjobs/q63539qt9yqaqspo/artifacts/YARA.OpenSSL.x64.1.1.1.nupkg" - VS: "Visual Studio 14 2015 Win64" - - PYTHON: "C:\\Python38" PYTHON_VERSION: "3.8.x" PYTHON_ARCH: "32" @@ -71,16 +55,29 @@ OPENSSL_LIB: "https://ci.appveyor.com/api/buildjobs/q63539qt9yqaqspo/artifacts/YARA.OpenSSL.x64.1.1.1.nupkg" VS: "Visual Studio 14 2015 Win64" + - PYTHON: "C:\\Python312" + PYTHON_VERSION: "3.12.0" + PYTHON_ARCH: "32" + OPENSSL_LIB: "https://ci.appveyor.com/api/buildjobs/fakubeldw67e9pmg/artifacts/YARA.OpenSSL.x86.1.1.1.nupkg" + VS: "Visual Studio 14 2015" + + - PYTHON: "C:\\Python312-x64" + PYTHON_VERSION: "3.12.0" + PYTHON_ARCH: "64" + OPENSSL_LIB: "https://ci.appveyor.com/api/buildjobs/q63539qt9yqaqspo/artifacts/YARA.OpenSSL.x64.1.1.1.nupkg" + VS: "Visual Studio 14 2015 Win64" + install: # If there is a newer build queued for the same PR, cancel this one. # The AppVeyor 'rollout builds' option is supposed to serve the same # purpose but it is problematic because it tends to cancel builds pushed # directly to master instead of just PR builds (or the converse). # credits: JuliaLang developers. - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } + - ps: + if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + throw "There are newer queued builds for this pull request, failing early." } - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" @@ -97,7 +94,7 @@ # Check that we have the expected version and architecture for Python - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - 'python -c "import struct; print(struct.calcsize(''P'') * 8)"' # Upgrade to the latest version of pip to avoid it displaying warnings # about it being out of date. @@ -111,6 +108,9 @@ # Install wheel. - "%CMD_IN_ENV% pip install wheel" + # Install setuptools. Setup tools doesn't comes by default in Python 3.12. + - "%CMD_IN_ENV% pip install setuptools" + # We are in projects/yara-python, lets go out to projects. - cd .. @@ -131,18 +131,16 @@ # projects/yara-python - cd ../../yara-python - clone_script: -- cmd: git clone -q --recursive --branch=%APPVEYOR_REPO_BRANCH% https://github.com/%APPVEYOR_REPO_NAME%.git %APPVEYOR_BUILD_FOLDER% -- cmd: git fetch -- cmd: git checkout -qf %APPVEYOR_REPO_BRANCH% + - cmd: git clone -q --recursive --branch=%APPVEYOR_REPO_BRANCH% https://github.com/%APPVEYOR_REPO_NAME%.git %APPVEYOR_BUILD_FOLDER% + - cmd: git fetch + - cmd: git checkout -qf %APPVEYOR_REPO_BRANCH% build_script: # Build the compiled extension - - "%CMD_IN_ENV% python setup.py build_ext --enable-cuckoo + - "%CMD_IN_ENV% python setup.py build_ext --enable-cuckoo --enable-openssl -L../jansson-%JANSSON_VERSION%/build/lib/Release;../openssl/lib -I../jansson-%JANSSON_VERSION%/build/include;../openssl/include - -DHASH_MODULE,HAVE_LIBCRYPTO,BUCKETS_128,CHECKSUM_1B -llibcrypto" after_build: @@ -164,10 +162,10 @@ provider: GitHub auth_token: secure: d3qqX7bmrBiKJI38yFPc5vHrGGfS3LxLC7FaG6ewI2ghPPE22Pk6QtyrEFFb73PL - artifact: /.*\.exe/ + artifact: /.*\.exe/ draft: true on: - APPVEYOR_REPO_TAG: true # deploy on tag push only + APPVEYOR_REPO_TAG: true # deploy on tag push only #on_success: # - TODO: upload the content of dist/*.whl to a public wheelhouse diff -Nru yara-python-4.3.1/debian/changelog yara-python-4.5.0/debian/changelog --- yara-python-4.3.1/debian/changelog 2023-10-31 18:04:31.000000000 +0000 +++ yara-python-4.5.0/debian/changelog 2024-02-14 12:36:24.000000000 +0000 @@ -1,8 +1,8 @@ -yara-python (4.3.1-1build1) noble; urgency=medium +yara-python (4.5.0-1) unstable; urgency=medium - * No-change rebuild with Python 3.12 as supported version + * New upstream version 4.5.0 - -- Graham Inggs Tue, 31 Oct 2023 18:04:31 +0000 + -- Hilko Bengen Wed, 14 Feb 2024 13:36:24 +0100 yara-python (4.3.1-1) unstable; urgency=medium diff -Nru yara-python-4.3.1/setup.py yara-python-4.5.0/setup.py --- yara-python-4.3.1/setup.py 2023-04-21 08:30:15.000000000 +0000 +++ yara-python-4.5.0/setup.py 2024-02-14 09:10:08.000000000 +0000 @@ -36,7 +36,9 @@ ('enable-magic', None, 'enable "magic" module'), ('enable-dex', None, 'enable "dex" module'), ('enable-macho', None, 'enable "macho" module'), - ('enable-profiling', None, 'enable profiling features')] + ('enable-profiling', None, 'enable profiling features'), + ('enable-openssl', None, 'enable features that depend on OpenSSL'), +] BOOLEAN_OPTIONS = [ @@ -45,7 +47,9 @@ 'enable-magic', 'enable-dex', 'enable-macho', - 'enable-profiling'] + 'enable-profiling', + 'enable-openssl', +] @contextlib.contextmanager @@ -120,6 +124,7 @@ self.enable_dex = None self.enable_macho = None self.enable_profiling = None + self.enable_openssl = None def finalize_options(self): @@ -141,6 +146,7 @@ self.enable_dex = None self.enable_macho = None self.enable_profiling = None + self.enable_openssl = None def finalize_options(self): @@ -155,7 +161,8 @@ ('enable_cuckoo', 'enable_cuckoo'), ('enable_dex', 'enable_dex'), ('enable_macho', 'enable_macho'), - ('enable_profiling', 'enable_profiling')) + ('enable_profiling', 'enable_profiling'), + ('enable_openssl', 'enable_openssl')) if self.enable_magic and self.dynamic_linking: raise distutils.errors.DistutilsOptionError( @@ -169,6 +176,10 @@ if self.enable_macho and self.dynamic_linking: raise distutils.errors.DistutilsOptionError( '--enable-macho can''t be used with --dynamic-linking') + if self.enable_openssl and self.dynamic_linking: + raise distutils.errors.DistutilsOptionError( + '--enable-enable-openssl can''t be used with --dynamic-linking') + def run(self): """Execute the build command.""" @@ -231,6 +242,8 @@ module.include_dirs.append('/usr/local/include') module.library_dirs.append('/usr/local/lib') module.library_dirs.append('/usr/local/opt/openssl/lib') + module.include_dirs.append('/opt/homebrew/include') + module.library_dirs.append('/opt/homebrew/opt/openssl/lib') elif building_for_freebsd: module.define_macros.append(('_GNU_SOURCE', '1')) module.define_macros.append(('USE_FREEBSD_PROC', '1')) @@ -279,20 +292,23 @@ include_dirs=module.include_dirs + openssl_include_dirs, libraries=module.libraries + openssl_libraries + ['dl', 'pthread', 'z'], library_dirs=module.library_dirs + openssl_library_dirs) - ): + or self.enable_openssl): module.define_macros.append(('HASH_MODULE', '1')) module.define_macros.append(('HAVE_LIBCRYPTO', '1')) module.libraries.extend(openssl_libraries) module.include_dirs.extend(openssl_include_dirs) module.library_dirs.extend(openssl_library_dirs) elif building_for_windows: - # OpenSSL is not available, but in Windows we can rely on Wincrypt. + # OpenSSL is not available, but in Windows we can rely on Wincrypt for + # hashing functions. module.define_macros.append(('HASH_MODULE', '1')) module.define_macros.append(('HAVE_WINCRYPT_H', '1')) + # The authenticode parser depends on OpenSSL and must be excluded. + exclusions.append('yara/libyara/modules/pe/authenticode-parser') else: - # OpenSSL is not available, exclude hash.c, as it requires some hashing - # functions. + # Without OpenSSL there's no hash module nor authenticode parser. exclusions.append('yara/libyara/modules/hash/hash.c') + exclusions.append('yara/libyara/modules/pe/authenticode-parser') if self.enable_magic: module.define_macros.append(('MAGIC_MODULE', '1')) @@ -326,10 +342,15 @@ exclusions = [os.path.normpath(x) for x in exclusions] for directory, _, files in os.walk('yara/libyara/'): - for x in files: - x = os.path.normpath(os.path.join(directory, x)) - if x.endswith('.c') and x not in exclusions: - module.sources.append(x) + for f in files: + f = os.path.normpath(os.path.join(directory, f)) + # Ignore any file that is not a .c file + if not f.endswith('.c'): + continue + # Ignore files that are listed in the exclusion list. + if any(map(lambda e: f.startswith(e), exclusions)): + continue + module.sources.append(f) build_ext.run(self) @@ -372,9 +393,10 @@ setup( name='yara-python', - version='4.3.1', + version='4.5.0', description='Python interface for YARA', long_description=readme, + long_description_content_type='text/markdown', license='Apache 2.0', author='Victor M. Alvarez', author_email='plusvic@gmail.com, vmalvarez@virustotal.com', diff -Nru yara-python-4.3.1/tests.py yara-python-4.5.0/tests.py --- yara-python-4.3.1/tests.py 2023-04-21 08:30:15.000000000 +0000 +++ yara-python-4.5.0/tests.py 2024-02-14 09:10:08.000000000 +0000 @@ -719,7 +719,7 @@ 'rule test { strings: $a = /[M-N]iss/ nocase condition: $a }', 'rule test { strings: $a = /(Mi|ssi)ssippi/ nocase condition: $a }', r'rule test { strings: $a = /ppi\tmi/ condition: $a }', - 'rule test { strings: $a = /ppi\.mi/ condition: $a }', + r'rule test { strings: $a = /ppi\.mi/ condition: $a }', 'rule test { strings: $a = /^mississippi/ fullword condition: $a }', 'rule test { strings: $a = /mississippi.*mississippi$/s condition: $a }', ], 'mississippi\tmississippi.mississippi\nmississippi') diff -Nru yara-python-4.3.1/yara-python.c yara-python-4.5.0/yara-python.c --- yara-python-4.3.1/yara-python.c 2023-04-21 08:30:15.000000000 +0000 +++ yara-python-4.5.0/yara-python.c 2024-02-14 09:10:08.000000000 +0000 @@ -1394,29 +1394,28 @@ PyObject* bytes = PyObject_CallMethod( (PyObject*) user_data, "read", "n", (Py_ssize_t) size); - PyGILState_Release(gil_state); - - if (bytes != NULL) + if (bytes == NULL) { - Py_ssize_t len; - char* buffer; - - int result = PyBytes_AsStringAndSize(bytes, &buffer, &len); + PyGILState_Release(gil_state); + return i; + } - if (result == -1 || (size_t) len < size) - { - Py_DECREF(bytes); - return i; - } + Py_ssize_t len; + char* buffer; - memcpy((char*) ptr + i * size, buffer, size); + int result = PyBytes_AsStringAndSize(bytes, &buffer, &len); - Py_DECREF(bytes); - } - else + if (result == -1 || (size_t) len < size) { + Py_DECREF(bytes); + PyGILState_Release(gil_state); return i; } + + memcpy((char*) ptr + i * size, buffer, size); + + Py_DECREF(bytes); + PyGILState_Release(gil_state); } return count; @@ -1444,12 +1443,11 @@ (PyObject*) user_data, "write", "s#", (char*) ptr + i * size, size); #endif + Py_XDECREF(result); PyGILState_Release(gil_state); if (result == NULL) return i; - - Py_DECREF(result); } return count; @@ -1886,6 +1884,7 @@ Py_DECREF(object->offset); Py_DECREF(object->matched_data); + Py_DECREF(object->matched_length); Py_DECREF(object->xor_key); PyObject_Del(self); @@ -1925,7 +1924,10 @@ StringMatchInstance* instance = (StringMatchInstance*) self; uint64_t xor_key = PyLong_AsUnsignedLongLong(instance->xor_key); if (xor_key == 0) + { + Py_INCREF(instance->matched_data); return instance->matched_data; + } int result = PyBytes_AsStringAndSize(instance->matched_data, &pb, &length); if (result == -1) @@ -2626,7 +2628,7 @@ { static char *kwlist[] = { "filepath", "source", "file", "filepaths", "sources", - "includes", "externals", "error_on_warning", "include_callback", NULL}; + "includes", "externals", "error_on_warning", "strict_escape", "include_callback", NULL}; YR_COMPILER* compiler; YR_RULES* yara_rules; @@ -2643,6 +2645,7 @@ PyObject* includes = NULL; PyObject* externals = NULL; PyObject* error_on_warning = NULL; + PyObject* strict_escape = NULL; PyObject* include_callback = NULL; Py_ssize_t pos = 0; @@ -2659,7 +2662,7 @@ if (PyArg_ParseTupleAndKeywords( args, keywords, - "|ssOOOOOOO", + "|ssOOOOOOOO", kwlist, &filepath, &source, @@ -2669,6 +2672,7 @@ &includes, &externals, &error_on_warning, + &strict_escape, &include_callback)) { char num_args = 0; @@ -2718,6 +2722,21 @@ } } + if (strict_escape != NULL) + { + if (PyBool_Check(strict_escape)) + { + compiler->strict_escape = PyObject_IsTrue(strict_escape); + } + else + { + yr_compiler_destroy(compiler); + return PyErr_Format( + PyExc_TypeError, + "'strict_escape' param must be of boolean type"); + } + } + if (includes != NULL) { if (PyBool_Check(includes))