diff -Nru spead2-1.10.0/3rdparty/pybind11/.appveyor.yml spead2-2.1.0/3rdparty/pybind11/.appveyor.yml --- spead2-1.10.0/3rdparty/pybind11/.appveyor.yml 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/.appveyor.yml 2019-10-08 08:35:40.000000000 +0000 @@ -3,6 +3,7 @@ - Visual Studio 2017 - Visual Studio 2015 test: off +skip_branch_with_pr: true build: parallel: true platform: @@ -41,13 +42,14 @@ if ($env:PYTHON) { if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" - python -m pip install --upgrade pip wheel - python -m pip install pytest numpy --no-warn-script-location + python -W ignore -m pip install --upgrade pip wheel + python -W ignore -m pip install pytest numpy --no-warn-script-location } elseif ($env:CONDA) { if ($env:CONDA -eq "27") { $env:CONDA = "" } if ($env:PLATFORM -eq "x64") { $env:CONDA = "$env:CONDA-x64" } $env:PATH = "C:\Miniconda$env:CONDA\;C:\Miniconda$env:CONDA\Scripts\;$env:PATH" $env:PYTHONHOME = "C:\Miniconda$env:CONDA" + conda --version conda install -y -q pytest numpy scipy } - ps: | @@ -60,6 +62,7 @@ -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DCMAKE_SUPPRESS_REGENERATION=1 + . - set MSBuildLogger="C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" - cmake --build . --config %CONFIG% --target pytest -- /m /v:m /logger:%MSBuildLogger% - cmake --build . --config %CONFIG% --target cpptest -- /m /v:m /logger:%MSBuildLogger% diff -Nru spead2-1.10.0/3rdparty/pybind11/CONTRIBUTING.md spead2-2.1.0/3rdparty/pybind11/CONTRIBUTING.md --- spead2-1.10.0/3rdparty/pybind11/CONTRIBUTING.md 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/CONTRIBUTING.md 2019-10-08 08:35:40.000000000 +0000 @@ -27,6 +27,8 @@ do add value by themselves. * Add tests for any new functionality and run the test suite (``make pytest``) to ensure that no existing features break. +* Please run ``flake8`` and ``tools/check-style.sh`` to check your code matches + the project style. (Note that ``check-style.sh`` requires ``gawk``.) * This project has a strong focus on providing general solutions using a minimal amount of code, thus small pull requests are greatly preferred. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/chrono.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/chrono.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/chrono.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/chrono.rst 2019-10-08 08:35:40.000000000 +0000 @@ -59,7 +59,7 @@ .. rubric:: Python to C++ -- ``datetime.datetime`` → ``std::chrono::system_clock::time_point`` +- ``datetime.datetime`` or ``datetime.date`` or ``datetime.time`` → ``std::chrono::system_clock::time_point`` Date/time objects are converted into system clock timepoints. Any timezone information is ignored and the type is treated as a naive object. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/eigen.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/eigen.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/eigen.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/eigen.rst 2019-10-08 08:35:40.000000000 +0000 @@ -37,7 +37,7 @@ data types are the same (e.g. ``dtype='float64'`` and ``MatrixType::Scalar`` is ``double``); and that the storage is layout compatible. The latter limitation is discussed in detail in the section below, and requires careful -consideration: by default, numpy matrices and eigen matrices are *not* storage +consideration: by default, numpy matrices and Eigen matrices are *not* storage compatible. If the numpy matrix cannot be used as is (either because its types differ, e.g. @@ -226,7 +226,7 @@ Failing rather than copying =========================== -The default behaviour when binding ``Eigen::Ref`` eigen +The default behaviour when binding ``Eigen::Ref`` Eigen references is to copy matrix values when passed a numpy array that does not conform to the element type of ``MatrixType`` or does not have a compatible stride layout. If you want to explicitly avoid copying in such a case, you @@ -289,13 +289,13 @@ will accept a row vector, it will be passed as a row vector. (The column vector takes precedence when both are supported, for example, when passing a 1D numpy array to a MatrixXd argument). Note that the type need not be -expicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an +explicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an Eigen ``Matrix``: you would end up with a 1x5 Eigen matrix. Passing the same to an ``Eigen::MatrixXd`` would result in a 5x1 Eigen matrix. -When returning an eigen vector to numpy, the conversion is ambiguous: a row +When returning an Eigen vector to numpy, the conversion is ambiguous: a row vector of length 4 could be returned as either a 1D array of length 4, or as a -2D array of size 1x4. When encoutering such a situation, pybind11 compromises +2D array of size 1x4. When encountering such a situation, pybind11 compromises by considering the returned Eigen type: if it is a compile-time vector--that is, the type has either the number of rows or columns set to 1 at compile time--pybind11 converts to a 1D numpy array when returning the value. For diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/overview.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/overview.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/overview.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/overview.rst 2019-10-08 08:35:40.000000000 +0000 @@ -131,6 +131,8 @@ +------------------------------------+---------------------------+-------------------------------+ | ``std::vector`` | STL dynamic array | :file:`pybind11/stl.h` | +------------------------------------+---------------------------+-------------------------------+ +| ``std::deque`` | STL double-ended queue | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-------------------------------+ | ``std::valarray`` | STL value array | :file:`pybind11/stl.h` | +------------------------------------+---------------------------+-------------------------------+ | ``std::list`` | STL linked list | :file:`pybind11/stl.h` | diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/stl.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/stl.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/cast/stl.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/cast/stl.rst 2019-10-08 08:35:40.000000000 +0000 @@ -5,7 +5,7 @@ ==================== When including the additional header file :file:`pybind11/stl.h`, conversions -between ``std::vector<>``/``std::list<>``/``std::array<>``, +between ``std::vector<>``/``std::deque<>``/``std::list<>``/``std::array<>``, ``std::set<>``/``std::unordered_set<>``, and ``std::map<>``/``std::unordered_map<>`` and the Python ``list``, ``set`` and ``dict`` data structures are automatically enabled. The types ``std::pair<>`` @@ -175,9 +175,6 @@ }, py::keep_alive<0, 1>()) /* Keep vector alive while iterator is used */ // .... -Please take a look at the :ref:`macro_notes` before using the -``PYBIND11_MAKE_OPAQUE`` macro. - .. seealso:: The file :file:`tests/test_opaque_types.cpp` contains a complete diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/classes.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/classes.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/classes.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/classes.rst 2019-10-08 08:35:40.000000000 +0000 @@ -46,11 +46,10 @@ .. code-block:: cpp PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); - animal + py::class_(m, "Animal") .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -81,10 +80,10 @@ } }; -The macro :func:`PYBIND11_OVERLOAD_PURE` should be used for pure virtual -functions, and :func:`PYBIND11_OVERLOAD` should be used for functions which have -a default implementation. There are also two alternate macros -:func:`PYBIND11_OVERLOAD_PURE_NAME` and :func:`PYBIND11_OVERLOAD_NAME` which +The macro :c:macro:`PYBIND11_OVERLOAD_PURE` should be used for pure virtual +functions, and :c:macro:`PYBIND11_OVERLOAD` should be used for functions which have +a default implementation. There are also two alternate macros +:c:macro:`PYBIND11_OVERLOAD_PURE_NAME` and :c:macro:`PYBIND11_OVERLOAD_NAME` which take a string-valued name argument between the *Parent class* and *Name of the function* slots, which defines the name of function in Python. This is required when the C++ and Python versions of the @@ -93,15 +92,14 @@ The binding code also needs a few minor adaptations (highlighted): .. code-block:: cpp - :emphasize-lines: 2,4,5 + :emphasize-lines: 2,3 PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); - animal + py::class_(m, "Animal") .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -116,11 +114,11 @@ Bindings should be made against the actual class, not the trampoline helper class. .. code-block:: cpp + :emphasize-lines: 3 - py::class_ animal(m, "Animal"); - animal - .def(py::init<>()) - .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ + py::class_(m, "Animal"); + .def(py::init<>()) + .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the @@ -155,9 +153,9 @@ .. code-block:: python - class Dachschund(Dog): + class Dachshund(Dog): def __init__(self, name): - Dog.__init__(self) # Without this, undefind behavior may occur if the C++ portions are referenced. + Dog.__init__(self) # Without this, undefined behavior may occur if the C++ portions are referenced. self.name = name def bark(self): return "yap!" @@ -241,7 +239,7 @@ class PyDog : public Dog { public: using Dog::Dog; // Inherit constructors - std::string go(int n_times) override { PYBIND11_OVERLOAD_PURE(std::string, Dog, go, n_times); } + std::string go(int n_times) override { PYBIND11_OVERLOAD(std::string, Dog, go, n_times); } std::string name() override { PYBIND11_OVERLOAD(std::string, Dog, name, ); } std::string bark() override { PYBIND11_OVERLOAD(std::string, Dog, bark, ); } }; @@ -327,6 +325,10 @@ Extended trampoline class functionality ======================================= +.. _extended_class_functionality_forced_trampoline: + +Forced trampoline class initialisation +-------------------------------------- The trampoline classes described in the previous sections are, by default, only initialized when needed. More specifically, they are initialized when a python class actually inherits from a registered type (instead of merely creating an @@ -354,6 +356,45 @@ See the file :file:`tests/test_virtual_functions.cpp` for complete examples showing both normal and forced trampoline instantiation. +Different method signatures +--------------------------- +The macro's introduced in :ref:`overriding_virtuals` cover most of the standard +use cases when exposing C++ classes to Python. Sometimes it is hard or unwieldy +to create a direct one-on-one mapping between the arguments and method return +type. + +An example would be when the C++ signature contains output arguments using +references (See also :ref:`faq_reference_arguments`). Another way of solving +this is to use the method body of the trampoline class to do conversions to the +input and return of the Python method. + +The main building block to do so is the :func:`get_overload`, this function +allows retrieving a method implemented in Python from within the trampoline's +methods. Consider for example a C++ method which has the signature +``bool myMethod(int32_t& value)``, where the return indicates whether +something should be done with the ``value``. This can be made convenient on the +Python side by allowing the Python function to return ``None`` or an ``int``: + +.. code-block:: cpp + + bool MyClass::myMethod(int32_t& value) + { + pybind11::gil_scoped_acquire gil; // Acquire the GIL while in this scope. + // Try to look up the overloaded method on the Python side. + pybind11::function overload = pybind11::get_overload(this, "myMethod"); + if (overload) { // method is found + auto obj = overload(value); // Call the Python function. + if (py::isinstance(obj)) { // check if it returned a Python integer type + value = obj.cast(); // Cast it and assign it to the value. + return true; // Return true; value should be used. + } else { + return false; // Python returned none, return false. + } + } + return false; // Alternatively return MyClass::myMethod(value); + } + + .. _custom_constructors: Custom constructors @@ -621,6 +662,7 @@ .def(py::self *= float()) .def(float() * py::self) .def(py::self * float()) + .def(-py::self) .def("__repr__", &Vector2::toString); } @@ -760,7 +802,7 @@ requirement is that the first template argument is the type to be declared. It is also permitted to inherit multiply from exported C++ classes in Python, -as well as inheriting from multiple Python and/or pybind-exported classes. +as well as inheriting from multiple Python and/or pybind11-exported classes. There is one caveat regarding the implementation of this feature: @@ -781,7 +823,7 @@ Module-local class bindings =========================== -When creating a binding for a class, pybind by default makes that binding +When creating a binding for a class, pybind11 by default makes that binding "global" across modules. What this means is that a type defined in one module can be returned from any module resulting in the same Python type. For example, this allows the following: @@ -999,3 +1041,86 @@ requires a more explicit function binding in the form of ``.def("foo", static_cast(&Publicist::foo));`` where ``int (A::*)() const`` is the type of ``A::foo``. + +Custom automatic downcasters +============================ + +As explained in :ref:`inheritance`, pybind11 comes with built-in +understanding of the dynamic type of polymorphic objects in C++; that +is, returning a Pet to Python produces a Python object that knows it's +wrapping a Dog, if Pet has virtual methods and pybind11 knows about +Dog and this Pet is in fact a Dog. Sometimes, you might want to +provide this automatic downcasting behavior when creating bindings for +a class hierarchy that does not use standard C++ polymorphism, such as +LLVM [#f4]_. As long as there's some way to determine at runtime +whether a downcast is safe, you can proceed by specializing the +``pybind11::polymorphic_type_hook`` template: + +.. code-block:: cpp + + enum class PetKind { Cat, Dog, Zebra }; + struct Pet { // Not polymorphic: has no virtual methods + const PetKind kind; + int age = 0; + protected: + Pet(PetKind _kind) : kind(_kind) {} + }; + struct Dog : Pet { + Dog() : Pet(PetKind::Dog) {} + std::string sound = "woof!"; + std::string bark() const { return sound; } + }; + + namespace pybind11 { + template<> struct polymorphic_type_hook { + static const void *get(const Pet *src, const std::type_info*& type) { + // note that src may be nullptr + if (src && src->kind == PetKind::Dog) { + type = &typeid(Dog); + return static_cast(src); + } + return src; + } + }; + } // namespace pybind11 + +When pybind11 wants to convert a C++ pointer of type ``Base*`` to a +Python object, it calls ``polymorphic_type_hook::get()`` to +determine if a downcast is possible. The ``get()`` function should use +whatever runtime information is available to determine if its ``src`` +parameter is in fact an instance of some class ``Derived`` that +inherits from ``Base``. If it finds such a ``Derived``, it sets ``type += &typeid(Derived)`` and returns a pointer to the ``Derived`` object +that contains ``src``. Otherwise, it just returns ``src``, leaving +``type`` at its default value of nullptr. If you set ``type`` to a +type that pybind11 doesn't know about, no downcasting will occur, and +the original ``src`` pointer will be used with its static type +``Base*``. + +It is critical that the returned pointer and ``type`` argument of +``get()`` agree with each other: if ``type`` is set to something +non-null, the returned pointer must point to the start of an object +whose type is ``type``. If the hierarchy being exposed uses only +single inheritance, a simple ``return src;`` will achieve this just +fine, but in the general case, you must cast ``src`` to the +appropriate derived-class pointer (e.g. using +``static_cast(src)``) before allowing it to be returned as a +``void*``. + +.. [#f4] https://llvm.org/docs/HowToSetUpLLVMStyleRTTI.html + +.. note:: + + pybind11's standard support for downcasting objects whose types + have virtual methods is implemented using + ``polymorphic_type_hook`` too, using the standard C++ ability to + determine the most-derived type of a polymorphic object using + ``typeid()`` and to cast a base pointer to that most-derived type + (even if you don't know what it is) using ``dynamic_cast``. + +.. seealso:: + + The file :file:`tests/test_tagbased_polymorphic.cpp` contains a + more complete example, including a demonstration of how to provide + automatic downcasting for an entire class hierarchy without + writing one get() function for each class. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/exceptions.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/exceptions.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/exceptions.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/exceptions.rst 2019-10-08 08:35:40.000000000 +0000 @@ -11,45 +11,45 @@ .. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}| -+--------------------------------------+------------------------------+ -| C++ exception type | Python exception type | -+======================================+==============================+ -| :class:`std::exception` | ``RuntimeError`` | -+--------------------------------------+------------------------------+ -| :class:`std::bad_alloc` | ``MemoryError`` | -+--------------------------------------+------------------------------+ -| :class:`std::domain_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::invalid_argument` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::length_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::out_of_range` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`std::range_error` | ``ValueError`` | -+--------------------------------------+------------------------------+ -| :class:`pybind11::stop_iteration` | ``StopIteration`` (used to | -| | implement custom iterators) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::index_error` | ``IndexError`` (used to | -| | indicate out of bounds | -| | accesses in ``__getitem__``, | -| | ``__setitem__``, etc.) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::value_error` | ``ValueError`` (used to | -| | indicate wrong value passed | -| | in ``container.remove(...)`` | -+--------------------------------------+------------------------------+ -| :class:`pybind11::key_error` | ``KeyError`` (used to | -| | indicate out of bounds | -| | accesses in ``__getitem__``, | -| | ``__setitem__`` in dict-like | -| | objects, etc.) | -+--------------------------------------+------------------------------+ -| :class:`pybind11::error_already_set` | Indicates that the Python | -| | exception flag has already | -| | been initialized | -+--------------------------------------+------------------------------+ ++--------------------------------------+--------------------------------------+ +| C++ exception type | Python exception type | ++======================================+======================================+ +| :class:`std::exception` | ``RuntimeError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::bad_alloc` | ``MemoryError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::domain_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::invalid_argument` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::length_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::out_of_range` | ``IndexError`` | ++--------------------------------------+--------------------------------------+ +| :class:`std::range_error` | ``ValueError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::stop_iteration` | ``StopIteration`` (used to implement | +| | custom iterators) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::index_error` | ``IndexError`` (used to indicate out | +| | of bounds access in ``__getitem__``, | +| | ``__setitem__``, etc.) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::value_error` | ``ValueError`` (used to indicate | +| | wrong value passed in | +| | ``container.remove(...)``) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::key_error` | ``KeyError`` (used to indicate out | +| | of bounds access in ``__getitem__``, | +| | ``__setitem__`` in dict-like | +| | objects, etc.) | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::error_already_set` | Indicates that the Python exception | +| | flag has already been set via Python | +| | API calls from C++ code; this C++ | +| | exception is used to propagate such | +| | a Python exception back to Python. | ++--------------------------------------+--------------------------------------+ When a Python function invoked from C++ throws an exception, it is converted into a C++ exception of type :class:`error_already_set` whose string payload diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/functions.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/functions.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/functions.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/functions.rst 2019-10-08 08:35:40.000000000 +0000 @@ -438,7 +438,7 @@ py::class_(m, "Cat").def(py::init<>()); m.def("bark", [](Dog *dog) -> std::string { if (dog) return "woof!"; /* Called with a Dog instance */ - else return "(no dog)"; /* Called with None, d == nullptr */ + else return "(no dog)"; /* Called with None, dog == nullptr */ }, py::arg("dog").none(true)); m.def("meow", [](Cat *cat) -> std::string { // Can't be called with None argument @@ -467,6 +467,15 @@ The default behaviour when the tag is unspecified is to allow ``None``. +.. note:: + + Even when ``.none(true)`` is specified for an argument, ``None`` will be converted to a + ``nullptr`` *only* for custom and :ref:`opaque ` types. Pointers to built-in types + (``double *``, ``int *``, ...) and STL types (``std::vector *``, ...; if ``pybind11/stl.h`` + is included) are copied when converted to C++ (see :doc:`/advanced/cast/overview`) and will + not allow ``None`` as argument. To pass optional argument of these copied types consider + using ``std::optional`` + Overload resolution order ========================= diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/misc.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/misc.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/misc.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/misc.rst 2019-10-08 08:35:40.000000000 +0000 @@ -7,13 +7,32 @@ ========================================== pybind11 provides a few convenience macros such as -:func:`PYBIND11_MAKE_OPAQUE` and :func:`PYBIND11_DECLARE_HOLDER_TYPE`, and -``PYBIND11_OVERLOAD_*``. Since these are "just" macros that are evaluated -in the preprocessor (which has no concept of types), they *will* get confused -by commas in a template argument such as ``PYBIND11_OVERLOAD(MyReturnValue, myFunc)``. In this case, the preprocessor assumes that the comma indicates -the beginning of the next parameter. Use a ``typedef`` to bind the template to -another name and use it in the macro to avoid this problem. +:func:`PYBIND11_DECLARE_HOLDER_TYPE` and ``PYBIND11_OVERLOAD_*``. Since these +are "just" macros that are evaluated in the preprocessor (which has no concept +of types), they *will* get confused by commas in a template argument; for +example, consider: + +.. code-block:: cpp + + PYBIND11_OVERLOAD(MyReturnType, Class, func) + +The limitation of the C preprocessor interprets this as five arguments (with new +arguments beginning after each comma) rather than three. To get around this, +there are two alternatives: you can use a type alias, or you can wrap the type +using the ``PYBIND11_TYPE`` macro: + +.. code-block:: cpp + + // Version 1: using a type alias + using ReturnType = MyReturnType; + using ClassType = Class; + PYBIND11_OVERLOAD(ReturnType, ClassType, func); + + // Version 2: using the PYBIND11_TYPE macro: + PYBIND11_OVERLOAD(PYBIND11_TYPE(MyReturnType), + PYBIND11_TYPE(Class), func) + +The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. .. _gil: @@ -137,7 +156,7 @@ Note that pybind11 code compiled with hidden-by-default symbol visibility (e.g. via the command line flag ``-fvisibility=hidden`` on GCC/Clang), which is -required proper pybind11 functionality, can interfere with the ability to +required for proper pybind11 functionality, can interfere with the ability to access types defined in another extension module. Working around this requires manually exporting types that are accessed by multiple extension modules; pybind11 provides a macro to do just this: @@ -216,6 +235,21 @@ // Create a weak reference with a cleanup callback and initially leak it (void) py::weakref(m.attr("BaseClass"), cleanup_callback).release(); +.. note:: + + PyPy (at least version 5.9) does not garbage collect objects when the + interpreter exits. An alternative approach (which also works on CPython) is to use + the :py:mod:`atexit` module [#f7]_, for example: + + .. code-block:: cpp + + auto atexit = py::module::import("atexit"); + atexit.attr("register")(py::cpp_function([]() { + // perform cleanup here -- this function is called with the GIL held + })); + + .. [#f7] https://docs.python.org/3/library/atexit.html + Generating documentation using Sphinx ===================================== diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/advanced/pycpp/numpy.rst spead2-2.1.0/3rdparty/pybind11/docs/advanced/pycpp/numpy.rst --- spead2-1.10.0/3rdparty/pybind11/docs/advanced/pycpp/numpy.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/advanced/pycpp/numpy.rst 2019-10-08 08:35:40.000000000 +0000 @@ -41,7 +41,7 @@ py::format_descriptor::format(), /* Python struct-style format descriptor */ 2, /* Number of dimensions */ { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + { sizeof(float) * m.cols(), /* Strides (in bytes) for each index */ sizeof(float) } ); }); @@ -364,3 +364,23 @@ The file :file:`tests/test_numpy_array.cpp` contains additional examples demonstrating the use of this feature. + +Ellipsis +======== + +Python 3 provides a convenient ``...`` ellipsis notation that is often used to +slice multidimensional arrays. For instance, the following snippet extracts the +middle dimensions of a tensor with the first and last index set to zero. + +.. code-block:: python + + a = # a NumPy array + b = a[0, ..., 0] + +The function ``py::ellipsis()`` function can be used to perform the same +operation on the C++ side: + +.. code-block:: cpp + + py::array a = /* A NumPy array */; + py::array b = a[py::make_tuple(0, py::ellipsis(), 0)]; diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/changelog.rst spead2-2.1.0/3rdparty/pybind11/docs/changelog.rst --- spead2-1.10.0/3rdparty/pybind11/docs/changelog.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/changelog.rst 2019-10-11 09:23:26.000000000 +0000 @@ -6,6 +6,174 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning `_ policy. +v2.4.2 (Sep 21, 2019) +----------------------------------------------------- + +* Replaced usage of a C++14 only construct. `#1929 + `_. + +* Made an ifdef future-proof for Python >= 4. `f3109d + `_. + +v2.4.1 (Sep 20, 2019) +----------------------------------------------------- + +* Fixed a problem involving implicit conversion from enumerations to integers + on Python 3.8. `#1780 `_. + +v2.4.0 (Sep 19, 2019) +----------------------------------------------------- + +* Try harder to keep pybind11-internal data structures separate when there + are potential ABI incompatibilities. Fixes crashes that occurred when loading + multiple pybind11 extensions that were e.g. compiled by GCC (libstdc++) + and Clang (libc++). + `#1588 `_ and + `c9f5a `_. + +* Added support for ``__await__``, ``__aiter__``, and ``__anext__`` protocols. + `#1842 `_. + +* ``pybind11_add_module()``: don't strip symbols when compiling in + ``RelWithDebInfo`` mode. `#1980 + `_. + +* ``enum_``: Reproduce Python behavior when comparing against invalid values + (e.g. ``None``, strings, etc.). Add back support for ``__invert__()``. + `#1912 `_, + `#1907 `_. + +* List insertion operation for ``py::list``. + Added ``.empty()`` to all collection types. + Added ``py::set::contains()`` and ``py::dict::contains()``. + `#1887 `_, + `#1884 `_, + `#1888 `_. + +* ``py::details::overload_cast_impl`` is available in C++11 mode, can be used + like ``overload_cast`` with an additional set of parantheses. + `#1581 `_. + +* Fixed ``get_include()`` on Conda. + `#1877 `_. + +* ``stl_bind.h``: negative indexing support. + `#1882 `_. + +* Minor CMake fix to add MinGW compatibility. + `#1851 `_. + +* GIL-related fixes. + `#1836 `_, + `8b90b `_. + +* Other very minor/subtle fixes and improvements. + `#1329 `_, + `#1910 `_, + `#1863 `_, + `#1847 `_, + `#1890 `_, + `#1860 `_, + `#1848 `_, + `#1821 `_, + `#1837 `_, + `#1833 `_, + `#1748 `_, + `#1852 `_. + +v2.3.0 (June 11, 2019) +----------------------------------------------------- + +* Significantly reduced module binary size (10-20%) when compiled in C++11 mode + with GCC/Clang, or in any mode with MSVC. Function signatures are now always + precomputed at compile time (this was previously only available in C++14 mode + for non-MSVC compilers). + `#934 `_. + +* Add basic support for tag-based static polymorphism, where classes + provide a method to returns the desired type of an instance. + `#1326 `_. + +* Python type wrappers (``py::handle``, ``py::object``, etc.) + now support map Python's number protocol onto C++ arithmetic + operators such as ``operator+``, ``operator/=``, etc. + `#1511 `_. + +* A number of improvements related to enumerations: + + 1. The ``enum_`` implementation was rewritten from scratch to reduce + code bloat. Rather than instantiating a full implementation for each + enumeration, most code is now contained in a generic base class. + `#1511 `_. + + 2. The ``value()`` method of ``py::enum_`` now accepts an optional + docstring that will be shown in the documentation of the associated + enumeration. `#1160 `_. + + 3. check for already existing enum value and throw an error if present. + `#1453 `_. + +* Support for over-aligned type allocation via C++17's aligned ``new`` + statement. `#1582 `_. + +* Added ``py::ellipsis()`` method for slicing of multidimensional NumPy arrays + `#1502 `_. + +* Numerous Improvements to the ``mkdoc.py`` script for extracting documentation + from C++ header files. + `#1788 `_. + +* ``pybind11_add_module()``: allow including Python as a ``SYSTEM`` include path. + `#1416 `_. + +* ``pybind11/stl.h`` does not convert strings to ``vector`` anymore. + `#1258 `_. + +* Mark static methods as such to fix auto-generated Sphinx documentation. + `#1732 `_. + +* Re-throw forced unwind exceptions (e.g. during pthread termination). + `#1208 `_. + +* Added ``__contains__`` method to the bindings of maps (``std::map``, + ``std::unordered_map``). + `#1767 `_. + +* Improvements to ``gil_scoped_acquire``. + `#1211 `_. + +* Type caster support for ``std::deque``. + `#1609 `_. + +* Support for ``std::unique_ptr`` holders, whose deleters differ between a base and derived + class. `#1353 `_. + +* Construction of STL array/vector-like data structures from + iterators. Added an ``extend()`` operation. + `#1709 `_, + +* CMake build system improvements for projects that include non-C++ + files (e.g. plain C, CUDA) in ``pybind11_add_module`` et al. + `#1678 `_. + +* Fixed asynchronous invocation and deallocation of Python functions + wrapped in ``std::function``. + `#1595 `_. + +* Fixes regarding return value policy propagation in STL type casters. + `#1603 `_. + +* Fixed scoped enum comparisons. + `#1571 `_. + +* Fixed iostream redirection for code that releases the GIL. + `#1368 `_, + +* A number of CI-related fixes. + `#1757 `_, + `#1744 `_, + `#1670 `_. + v2.2.4 (September 11, 2018) ----------------------------------------------------- @@ -14,7 +182,7 @@ `#1517 `_. * Fixes for newer MSVC versions and C++17 mode. - `#1347 `_. + `#1347 `_, `#1462 `_. * Propagate return value policies to type-specific casters @@ -50,6 +218,11 @@ * A few minor typo fixes and improvements to the test suite, and patches that silence compiler warnings. +* Vectors now support construction from generators, as well as ``extend()`` from a + list or generator. + `#1496 `_. + + v2.2.3 (April 29, 2018) ----------------------------------------------------- @@ -69,10 +242,6 @@ * Fixed an endianness-related fault in the test suite. `#1287 `_. -* Intel compilers have needed to be >= 17.0 since v2.1. Now the check - is explicit and a compile-time error is raised if the compiler does - not meet the requirements. - v2.2.2 (February 7, 2018) ----------------------------------------------------- @@ -363,6 +532,9 @@ * Fixed overriding static properties in derived classes. `#784 `_. +* Added support for write only properties. + `#1144 `_. + * Improved deduction of member functions of a derived class when its bases aren't registered with pybind11. `#855 `_. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/classes.rst spead2-2.1.0/3rdparty/pybind11/docs/classes.rst --- spead2-1.10.0/3rdparty/pybind11/docs/classes.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/classes.rst 2019-10-08 08:35:40.000000000 +0000 @@ -155,6 +155,9 @@ .def_property("name", &Pet::getName, &Pet::setName) // ... remainder ... +Write only properties can be defined by passing ``nullptr`` as the +input for the read function. + .. seealso:: Similar functions :func:`class_::def_readwrite_static`, @@ -225,8 +228,8 @@ .. _inheritance: -Inheritance and automatic upcasting -=================================== +Inheritance and automatic downcasting +===================================== Suppose now that the example consists of two data structures with an inheritance relationship: @@ -295,7 +298,7 @@ >>> p = example.pet_store() >>> type(p) # `Dog` instance behind `Pet` pointer - Pet # no pointer upcasting for regular non-polymorphic types + Pet # no pointer downcasting for regular non-polymorphic types >>> p.bark() AttributeError: 'Pet' object has no attribute 'bark' @@ -327,11 +330,11 @@ >>> p = example.pet_store2() >>> type(p) - PolymorphicDog # automatically upcast + PolymorphicDog # automatically downcast >>> p.bark() u'woof!' -Given a pointer to a polymorphic base, pybind11 performs automatic upcasting +Given a pointer to a polymorphic base, pybind11 performs automatic downcasting to the actual derived type. Note that this goes beyond the usual situation in C++: we don't just get access to the virtual functions of the base, we get the concrete derived type including functions and attributes that the base type may @@ -419,6 +422,17 @@ .def("foo_mutable", py::overload_cast(&Widget::foo)) .def("foo_const", py::overload_cast(&Widget::foo, py::const_)); +If you prefer the ``py::overload_cast`` syntax but have a C++11 compatible compiler only, +you can use ``py::detail::overload_cast_impl`` with an additional set of parentheses: + +.. code-block:: cpp + + template + using overload_cast_ = pybind11::detail::overload_cast_impl; + + py::class_(m, "Pet") + .def("set", overload_cast_()(&Pet::set), "Set the pet's age") + .def("set", overload_cast_()(&Pet::set), "Set the pet's name"); .. [#cpp14] A compiler which supports the ``-std=c++14`` flag or Visual Studio 2015 Update 2 and newer. @@ -485,6 +499,24 @@ >>> Pet.Kind.__members__ {'Dog': Kind.Dog, 'Cat': Kind.Cat} +The ``name`` property returns the name of the enum value as a unicode string. + +.. note:: + + It is also possible to use ``str(enum)``, however these accomplish different + goals. The following shows how these two approaches differ. + + .. code-block:: pycon + + >>> p = Pet( "Lucy", Pet.Cat ) + >>> pet_type = p.type + >>> pet_type + Pet.Cat + >>> str(pet_type) + 'Pet.Cat' + >>> pet_type.name + 'Cat' + .. note:: When the special tag ``py::arithmetic()`` is specified to the ``enum_`` diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/compiling.rst spead2-2.1.0/3rdparty/pybind11/docs/compiling.rst --- spead2-1.10.0/3rdparty/pybind11/docs/compiling.rst 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/compiling.rst 2019-10-08 08:35:40.000000000 +0000 @@ -59,7 +59,7 @@ .. code-block:: cmake pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] - [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) + [NO_EXTRAS] [SYSTEM] [THIN_LTO] source1 [source2 ...]) This function behaves very much like CMake's builtin ``add_library`` (in fact, it's a wrapper function around that command). It will add a library target @@ -86,6 +86,10 @@ given, they will always be disabled, even in ``Release`` mode. However, this will result in code bloat and is generally not recommended. +By default, pybind11 and Python headers will be included with ``-I``. In order +to include pybind11 as system library, e.g. to avoid warnings in downstream +code with warn-levels outside of pybind11's scope, set the option ``SYSTEM``. + As stated above, LTO is enabled by default. Some newer compilers also support different flavors of LTO such as `ThinLTO`_. Setting ``THIN_LTO`` will cause the function to prefer this flavor if available. The function falls back to @@ -145,6 +149,18 @@ find_package(pybind11 REQUIRED) pybind11_add_module(example example.cpp) +Note that ``find_package(pybind11)`` will only work correctly if pybind11 +has been correctly installed on the system, e. g. after downloading or cloning +the pybind11 repository : + +.. code-block:: bash + + cd pybind11 + mkdir build + cd build + cmake .. + make install + Once detected, the aforementioned ``pybind11_add_module`` can be employed as before. The function usage and configuration variables are identical no matter if pybind11 is added as a subdirectory or found as an installed package. You diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/conf.py spead2-2.1.0/3rdparty/pybind11/docs/conf.py --- spead2-1.10.0/3rdparty/pybind11/docs/conf.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/conf.py 2019-10-11 09:23:26.000000000 +0000 @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = '2.2' +version = '2.4' # The full version, including alpha/beta/rc tags. -release = '2.2.3' +release = '2.4.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/faq.rst spead2-2.1.0/3rdparty/pybind11/docs/faq.rst --- spead2-1.10.0/3rdparty/pybind11/docs/faq.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/faq.rst 2019-10-08 08:35:40.000000000 +0000 @@ -4,9 +4,13 @@ "ImportError: dynamic module does not define init function" =========================================================== -You are likely using an incompatible version of Python (for instance, the -extension library was compiled against Python 2, while the interpreter is -running on top of some version of Python 3, or vice versa). +1. Make sure that the name specified in PYBIND11_MODULE is identical to the +filename of the extension library (without prefixes such as .so) + +2. If the above did not fix the issue, you are likely using an incompatible +version of Python (for instance, the extension library was compiled against +Python 2, while the interpreter is running on top of some version of Python +3, or vice versa). "Symbol not found: ``__Py_ZeroStruct`` / ``_PyInstanceMethod_Type``" ======================================================================== @@ -35,6 +39,8 @@ cmake -DPYTHON_EXECUTABLE:FILEPATH= . +.. _faq_reference_arguments: + Limitations involving reference arguments ========================================= @@ -228,50 +234,53 @@ potential serious issues when loading multiple modules and is required for proper pybind operation. See the previous FAQ entry for more details. -Another aspect that can require a fair bit of code are function signature -descriptions. pybind11 automatically generates human-readable function -signatures for docstrings, e.g.: - -.. code-block:: none - - | __init__(...) - | __init__(*args, **kwargs) - | Overloaded function. - | - | 1. __init__(example.Example1) -> NoneType - | - | Docstring for overload #1 goes here - | - | 2. __init__(example.Example1, int) -> NoneType - | - | Docstring for overload #2 goes here - | - | 3. __init__(example.Example1, example.Example1) -> NoneType - | - | Docstring for overload #3 goes here - - -In C++11 mode, these are generated at run time using string concatenation, -which can amount to 10-20% of the size of the resulting binary. If you can, -enable C++14 language features (using ``-std=c++14`` for GCC/Clang), in which -case signatures are efficiently pre-generated at compile time. Unfortunately, -Visual Studio's C++14 support (``constexpr``) is not good enough as of April -2016, so it always uses the more expensive run-time approach. - -Working with ancient Visual Studio 2009 builds on Windows +Working with ancient Visual Studio 2008 builds on Windows ========================================================= The official Windows distributions of Python are compiled using truly ancient versions of Visual Studio that lack good C++11 support. Some users implicitly assume that it would be impossible to load a plugin built with Visual Studio 2015 into a Python distribution that was compiled using Visual -Studio 2009. However, no such issue exists: it's perfectly legitimate to +Studio 2008. However, no such issue exists: it's perfectly legitimate to interface DLLs that are built with different compilers and/or C libraries. Common gotchas to watch out for involve not ``free()``-ing memory region that that were ``malloc()``-ed in another shared library, using data structures with incompatible ABIs, and so on. pybind11 is very careful not to make these types of mistakes. +Inconsistent detection of Python version in CMake and pybind11 +============================================================== + +The functions ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` provided by CMake +for Python version detection are not used by pybind11 due to unreliability and limitations that make +them unsuitable for pybind11's needs. Instead pybind provides its own, more reliable Python detection +CMake code. Conflicts can arise, however, when using pybind11 in a project that *also* uses the CMake +Python detection in a system with several Python versions installed. + +This difference may cause inconsistencies and errors if *both* mechanisms are used in the same project. Consider the following +Cmake code executed in a system with Python 2.7 and 3.x installed: + +.. code-block:: cmake + + find_package(PythonInterp) + find_package(PythonLibs) + find_package(pybind11) + +It will detect Python 2.7 and pybind11 will pick it as well. + +In contrast this code: + +.. code-block:: cmake + + find_package(pybind11) + find_package(PythonInterp) + find_package(PythonLibs) + +will detect Python 3.x for pybind11 and may crash on ``find_package(PythonLibs)`` afterwards. + +It is advised to avoid using ``find_package(PythonInterp)`` and ``find_package(PythonLibs)`` from CMake and rely +on pybind11 in detecting Python version. If this is not possible CMake machinery should be called *before* including pybind11. + How to cite this project? ========================= @@ -286,4 +295,3 @@ note = {https://github.com/pybind/pybind11}, title = {pybind11 -- Seamless operability between C++11 and Python} } - diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/intro.rst spead2-2.1.0/3rdparty/pybind11/docs/intro.rst --- spead2-1.10.0/3rdparty/pybind11/docs/intro.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/intro.rst 2019-10-08 08:35:40.000000000 +0000 @@ -41,7 +41,6 @@ - Custom operators - Single and multiple inheritance - STL data structures -- Iterators and ranges - Smart pointers with reference counting like ``std::shared_ptr`` - Internal references with correct reference counting - C++ classes with virtual (and pure virtual) methods can be extended in Python @@ -77,9 +76,8 @@ of `PyRosetta`_, an enormous Boost.Python binding project, reported a binary size reduction of **5.4x** and compile time reduction by **5.8x**. -- When supported by the compiler, two new C++14 features (relaxed constexpr and - return value deduction) are used to precompute function signatures at compile - time, leading to smaller binaries. +- Function signatures are precomputed at compile time (using ``constexpr``), + leading to smaller binaries. - With little extra effort, C++ types can be pickled and unpickled similar to regular Python objects. diff -Nru spead2-1.10.0/3rdparty/pybind11/docs/reference.rst spead2-2.1.0/3rdparty/pybind11/docs/reference.rst --- spead2-1.10.0/3rdparty/pybind11/docs/reference.rst 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/docs/reference.rst 2019-10-08 08:35:40.000000000 +0000 @@ -86,6 +86,21 @@ .. doxygengroup:: python_builtins :members: +Inheritance +=========== + +See :doc:`/classes` and :doc:`/advanced/classes` for more detail. + +.. doxygendefine:: PYBIND11_OVERLOAD + +.. doxygendefine:: PYBIND11_OVERLOAD_PURE + +.. doxygendefine:: PYBIND11_OVERLOAD_NAME + +.. doxygendefine:: PYBIND11_OVERLOAD_PURE_NAME + +.. doxygenfunction:: get_overload + Exceptions ========== diff -Nru spead2-1.10.0/3rdparty/pybind11/.gitmodules spead2-2.1.0/3rdparty/pybind11/.gitmodules --- spead2-1.10.0/3rdparty/pybind11/.gitmodules 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/.gitmodules 2019-10-08 08:35:40.000000000 +0000 @@ -1,3 +1,3 @@ [submodule "tools/clang"] path = tools/clang - url = https://github.com/wjakob/clang-cindex-python3 + url = ../../wjakob/clang-cindex-python3 diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/attr.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/attr.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/attr.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/attr.h 2019-10-08 08:35:40.000000000 +0000 @@ -200,7 +200,8 @@ /// Special data structure which (temporarily) holds metadata about a bound class struct type_record { PYBIND11_NOINLINE type_record() - : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), module_local(false) { } + : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), + default_holder(true), module_local(false) { } /// Handle to the parent scope handle scope; @@ -214,11 +215,14 @@ /// How large is the underlying C++ type? size_t type_size = 0; + /// What is the alignment of the underlying C++ type? + size_t type_align = 0; + /// How large is the type's holder? size_t holder_size = 0; /// The global operator new can be overridden with a class-specific variant - void *(*operator_new)(size_t) = ::operator new; + void *(*operator_new)(size_t) = nullptr; /// Function pointer to class_<..>::init_instance void (*init_instance)(instance *, const void *) = nullptr; @@ -278,7 +282,7 @@ } }; -inline function_call::function_call(function_record &f, handle p) : +inline function_call::function_call(const function_record &f, handle p) : func(f), parent(p) { args.reserve(f.nargs); args_convert.reserve(f.nargs); diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/cast.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/cast.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/cast.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/cast.h 2019-10-08 08:35:40.000000000 +0000 @@ -17,6 +17,7 @@ #include #include #include +#include #if defined(PYBIND11_CPP17) # if defined(__has_include) @@ -203,10 +204,10 @@ } struct value_and_holder { - instance *inst; - size_t index; - const detail::type_info *type; - void **vh; + instance *inst = nullptr; + size_t index = 0u; + const detail::type_info *type = nullptr; + void **vh = nullptr; // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) : @@ -215,7 +216,7 @@ {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() : inst{nullptr} {} + value_and_holder() {} // Used for past-the-end iterator value_and_holder(size_t index) : index{index} {} @@ -269,8 +270,8 @@ struct iterator { private: - instance *inst; - const type_vec *types; + instance *inst = nullptr; + const type_vec *types = nullptr; value_and_holder curr; friend struct values_and_holders; iterator(instance *inst, const type_vec *tinfo) @@ -570,7 +571,17 @@ // Lazy allocation for unallocated values: if (vptr == nullptr) { auto *type = v_h.type ? v_h.type : typeinfo; - vptr = type->operator_new(type->type_size); + if (type->operator_new) { + vptr = type->operator_new(type->type_size); + } else { + #if defined(PYBIND11_CPP17) + if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) + vptr = ::operator new(type->type_size, + (std::align_val_t) type->type_align); + else + #endif + vptr = ::operator new(type->type_size); + } } value = vptr; } @@ -764,7 +775,9 @@ // so, copy constructability depends on whether the value_type is copy constructible. template struct is_copy_constructible, - std::is_same + std::is_same, + // Avoid infinite recursion + negation> >::value>> : is_copy_constructible {}; #if !defined(PYBIND11_CPP17) @@ -774,11 +787,47 @@ : all_of, is_copy_constructible> {}; #endif +NAMESPACE_END(detail) + +// polymorphic_type_hook::get(src, tinfo) determines whether the object pointed +// to by `src` actually is an instance of some class derived from `itype`. +// If so, it sets `tinfo` to point to the std::type_info representing that derived +// type, and returns a pointer to the start of the most-derived object of that type +// (in which `src` is a subobject; this will be the same address as `src` in most +// single inheritance cases). If not, or if `src` is nullptr, it simply returns `src` +// and leaves `tinfo` at its default value of nullptr. +// +// The default polymorphic_type_hook just returns src. A specialization for polymorphic +// types determines the runtime type of the passed object and adjusts the this-pointer +// appropriately via dynamic_cast. This is what enables a C++ Animal* to appear +// to Python as a Dog (if Dog inherits from Animal, Animal is polymorphic, Dog is +// registered with pybind11, and this Animal is in fact a Dog). +// +// You may specialize polymorphic_type_hook yourself for types that want to appear +// polymorphic to Python but do not use C++ RTTI. (This is a not uncommon pattern +// in performance-sensitive applications, used most notably in LLVM.) +template +struct polymorphic_type_hook +{ + static const void *get(const itype *src, const std::type_info*&) { return src; } +}; +template +struct polymorphic_type_hook::value>> +{ + static const void *get(const itype *src, const std::type_info*& type) { + type = src ? &typeid(*src) : nullptr; + return dynamic_cast(src); + } +}; + +NAMESPACE_BEGIN(detail) + /// Generic type caster for objects stored on the heap template class type_caster_base : public type_caster_generic { using itype = intrinsic_t; + public: - static PYBIND11_DESCR name() { return type_descr(_()); } + static constexpr auto name = _(); type_caster_base() : type_caster_base(typeid(type)) { } explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { } @@ -793,32 +842,28 @@ return cast(&src, return_value_policy::move, parent); } - // Returns a (pointer, type_info) pair taking care of necessary RTTI type lookup for a - // polymorphic type. If the instance isn't derived, returns the non-RTTI base version. - template ::value, int> = 0> + // Returns a (pointer, type_info) pair taking care of necessary type lookup for a + // polymorphic type (using RTTI by default, but can be overridden by specializing + // polymorphic_type_hook). If the instance isn't derived, returns the base version. static std::pair src_and_type(const itype *src) { - const void *vsrc = src; auto &cast_type = typeid(itype); const std::type_info *instance_type = nullptr; - if (vsrc) { - instance_type = &typeid(*src); - if (!same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type; if it is a pybind11-registered type, we - // can get the correct derived pointer (which may be != base pointer) by a - // dynamic_cast to most derived type: - if (auto *tpi = get_type_info(*instance_type)) - return {dynamic_cast(src), const_cast(tpi)}; - } + const void *vsrc = polymorphic_type_hook::get(src, instance_type); + if (instance_type && !same_type(cast_type, *instance_type)) { + // This is a base pointer to a derived type. If the derived type is registered + // with pybind11, we want to make the full derived object available. + // In the typical case where itype is polymorphic, we get the correct + // derived pointer (which may be != base pointer) by a dynamic_cast to + // most derived type. If itype is not polymorphic, we won't get here + // except via a user-provided specialization of polymorphic_type_hook, + // and the user has promised that no this-pointer adjustment is + // required in that case, so it's OK to use static_cast. + if (const auto *tpi = get_type_info(*instance_type)) + return {vsrc, tpi}; } // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so // don't do a cast - return type_caster_generic::src_and_type(vsrc, cast_type, instance_type); - } - - // Non-polymorphic type, so no dynamic casting; just call the generic version directly - template ::value, int> = 0> - static std::pair src_and_type(const itype *src) { - return type_caster_generic::src_and_type(src, typeid(itype)); + return type_caster_generic::src_and_type(src, cast_type, instance_type); } static handle cast(const itype *src, return_value_policy policy, handle parent) { @@ -835,7 +880,7 @@ nullptr, nullptr, holder); } - template using cast_op_type = cast_op_type; + template using cast_op_type = detail::cast_op_type; operator itype*() { return (type *) value; } operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); } @@ -885,7 +930,7 @@ "std::reference_wrapper caster requires T to have a caster with an `T &` operator"); public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } - static PYBIND11_DESCR name() { return caster_t::name(); } + static constexpr auto name = caster_t::name; static handle cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { // It is definitely wrong to take ownership of this pointer, so mask that rvp if (policy == return_value_policy::take_ownership || policy == return_value_policy::automatic) @@ -900,7 +945,7 @@ protected: \ type value; \ public: \ - static PYBIND11_DESCR name() { return type_descr(py_name); } \ + static constexpr auto name = py_name; \ template >::value, int> = 0> \ static handle cast(T_ *src, return_value_policy policy, handle parent) { \ if (!src) return none().release(); \ @@ -952,9 +997,11 @@ } bool py_err = py_value == (py_type) -1 && PyErr_Occurred(); + + // Protect std::numeric_limits::min/max with parentheses if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && - (py_value < (py_type) std::numeric_limits::min() || - py_value > (py_type) std::numeric_limits::max()))) { + (py_value < (py_type) (std::numeric_limits::min)() || + py_value > (py_type) (std::numeric_limits::max)()))) { bool type_error = py_err && PyErr_ExceptionMatches( #if PY_VERSION_HEX < 0x03000000 && !defined(PYPY_VERSION) PyExc_SystemError @@ -977,20 +1024,34 @@ return true; } - static handle cast(T src, return_value_policy /* policy */, handle /* parent */) { - if (std::is_floating_point::value) { - return PyFloat_FromDouble((double) src); - } else if (sizeof(T) <= sizeof(long)) { - if (std::is_signed::value) - return PyLong_FromLong((long) src); - else - return PyLong_FromUnsignedLong((unsigned long) src); - } else { - if (std::is_signed::value) - return PyLong_FromLongLong((long long) src); - else - return PyLong_FromUnsignedLongLong((unsigned long long) src); - } + template + static typename std::enable_if::value, handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyFloat_FromDouble((double) src); + } + + template + static typename std::enable_if::value && std::is_signed::value && (sizeof(U) <= sizeof(long)), handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_SIGNED((long) src); + } + + template + static typename std::enable_if::value && std::is_unsigned::value && (sizeof(U) <= sizeof(unsigned long)), handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_UNSIGNED((unsigned long) src); + } + + template + static typename std::enable_if::value && std::is_signed::value && (sizeof(U) > sizeof(long)), handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromLongLong((long long) src); + } + + template + static typename std::enable_if::value && std::is_unsigned::value && (sizeof(U) > sizeof(unsigned long)), handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromUnsignedLongLong((unsigned long long) src); } PYBIND11_TYPE_CASTER(T, _::value>("int", "float")); @@ -1049,7 +1110,7 @@ template using cast_op_type = void*&; operator void *&() { return value; } - static PYBIND11_DESCR name() { return type_descr(_("capsule")); } + static constexpr auto name = _("capsule"); private: void *value = nullptr; }; @@ -1292,7 +1353,7 @@ return one_char; } - static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); } + static constexpr auto name = _(PYBIND11_STRING_NAME); template using cast_op_type = pybind11::detail::cast_op_type<_T>; }; @@ -1317,9 +1378,7 @@ return cast_impl(std::forward(src), policy, parent, indices{}); } - static PYBIND11_DESCR name() { - return type_descr(_("Tuple[") + detail::concat(make_caster::name()...) + _("]")); - } + static constexpr auto name = _("Tuple[") + concat(make_caster::name...) + _("]"); template using cast_op_type = type; @@ -1464,7 +1523,7 @@ auto *ptr = holder_helper::get(src); return type_caster_base::cast_holder(ptr, std::addressof(src)); } - static PYBIND11_DESCR name() { return type_caster_base::name(); } + static constexpr auto name = type_caster_base::name; }; template @@ -1495,10 +1554,10 @@ template struct is_holder_type> : std::true_type {}; -template struct handle_type_name { static PYBIND11_DESCR name() { return _(); } }; -template <> struct handle_type_name { static PYBIND11_DESCR name() { return _(PYBIND11_BYTES_NAME); } }; -template <> struct handle_type_name { static PYBIND11_DESCR name() { return _("*args"); } }; -template <> struct handle_type_name { static PYBIND11_DESCR name() { return _("**kwargs"); } }; +template struct handle_type_name { static constexpr auto name = _(); }; +template <> struct handle_type_name { static constexpr auto name = _(PYBIND11_BYTES_NAME); }; +template <> struct handle_type_name { static constexpr auto name = _("*args"); }; +template <> struct handle_type_name { static constexpr auto name = _("**kwargs"); }; template struct pyobject_caster { @@ -1516,7 +1575,7 @@ static handle cast(const handle &src, return_value_policy /* policy */, handle /* parent */) { return src.inc_ref(); } - PYBIND11_TYPE_CASTER(type, handle_type_name::name()); + PYBIND11_TYPE_CASTER(type, handle_type_name::name); }; template @@ -1556,7 +1615,8 @@ // everything else returns a reference/pointer to a local variable. template using cast_is_temporary_value_reference = bool_constant< (std::is_reference::value || std::is_pointer::value) && - !std::is_base_of>::value + !std::is_base_of>::value && + !std::is_same, void>::value >; // When a value returned from a C++ function is being cast back to Python, we almost always want to @@ -1569,8 +1629,9 @@ template struct return_value_policy_override>::value, void>> { static return_value_policy policy(return_value_policy p) { - return !std::is_lvalue_reference::value && !std::is_pointer::value - ? return_value_policy::move : p; + return !std::is_lvalue_reference::value && + !std::is_pointer::value + ? return_value_policy::move : p; } }; @@ -1798,7 +1859,7 @@ /// Internal data associated with a single function call struct function_call { - function_call(function_record &f, handle p); // Implementation in attr.h + function_call(const function_record &f, handle p); // Implementation in attr.h /// The function data: const function_record &func; @@ -1840,7 +1901,7 @@ static constexpr bool has_kwargs = kwargs_pos < 0; static constexpr bool has_args = args_pos < 0; - static PYBIND11_DESCR arg_names() { return detail::concat(make_caster::name()...); } + static constexpr auto arg_names = concat(type_descr(make_caster::name)...); bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); @@ -2059,9 +2120,13 @@ NAMESPACE_END(detail) -#define PYBIND11_MAKE_OPAQUE(Type) \ +#define PYBIND11_MAKE_OPAQUE(...) \ namespace pybind11 { namespace detail { \ - template<> class type_caster : public type_caster_base { }; \ + template<> class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> { }; \ }} +/// Lets you pass a type containing a `,` through a macro parameter without needing a separate +/// typedef, e.g.: `PYBIND11_OVERLOAD(PYBIND11_TYPE(ReturnType), PYBIND11_TYPE(Parent), f, arg)` +#define PYBIND11_TYPE(...) __VA_ARGS__ + NAMESPACE_END(PYBIND11_NAMESPACE) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/chrono.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/chrono.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/chrono.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/chrono.h 2019-10-08 08:35:40.000000000 +0000 @@ -106,8 +106,11 @@ if (!PyDateTimeAPI) { PyDateTime_IMPORT; } if (!src) return false; + + std::tm cal; + microseconds msecs; + if (PyDateTime_Check(src.ptr())) { - std::tm cal; cal.tm_sec = PyDateTime_DATE_GET_SECOND(src.ptr()); cal.tm_min = PyDateTime_DATE_GET_MINUTE(src.ptr()); cal.tm_hour = PyDateTime_DATE_GET_HOUR(src.ptr()); @@ -115,11 +118,30 @@ cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; cal.tm_isdst = -1; - - value = system_clock::from_time_t(std::mktime(&cal)) + microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); - return true; + msecs = microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); + } else if (PyDate_Check(src.ptr())) { + cal.tm_sec = 0; + cal.tm_min = 0; + cal.tm_hour = 0; + cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); + cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; + cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + cal.tm_isdst = -1; + msecs = microseconds(0); + } else if (PyTime_Check(src.ptr())) { + cal.tm_sec = PyDateTime_TIME_GET_SECOND(src.ptr()); + cal.tm_min = PyDateTime_TIME_GET_MINUTE(src.ptr()); + cal.tm_hour = PyDateTime_TIME_GET_HOUR(src.ptr()); + cal.tm_mday = 1; // This date (day, month, year) = (1, 0, 70) + cal.tm_mon = 0; // represents 1-Jan-1970, which is the first + cal.tm_year = 70; // earliest available date for Python's datetime + cal.tm_isdst = -1; + msecs = microseconds(PyDateTime_TIME_GET_MICROSECOND(src.ptr())); } else return false; + + value = system_clock::from_time_t(std::mktime(&cal)) + msecs; + return true; } static handle cast(const std::chrono::time_point &src, return_value_policy /* policy */, handle /* parent */) { @@ -128,7 +150,7 @@ // Lazy initialise the PyDateTime import if (!PyDateTimeAPI) { PyDateTime_IMPORT; } - std::time_t tt = system_clock::to_time_t(src); + std::time_t tt = system_clock::to_time_t(time_point_cast(src)); // this function uses static memory so it's best to copy it out asap just in case // otherwise other code that is using localtime may break this (not just python code) std::tm localtime = *std::localtime(&tt); diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/complex.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/complex.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/complex.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/complex.h 2019-10-08 08:35:40.000000000 +0000 @@ -25,9 +25,13 @@ static std::string format() { return std::string(value); } }; +#ifndef PYBIND11_CPP17 + template constexpr const char format_descriptor< std::complex, detail::enable_if_t::value>>::value[3]; +#endif + NAMESPACE_BEGIN(detail) template struct is_fmt_numeric, detail::enable_if_t::value>> { diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/class.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/class.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/class.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/class.h 2019-10-08 08:35:40.000000000 +0000 @@ -10,6 +10,7 @@ #pragma once #include "../attr.h" +#include "../options.h" NAMESPACE_BEGIN(PYBIND11_NAMESPACE) NAMESPACE_BEGIN(detail) @@ -289,13 +290,9 @@ inline void add_patient(PyObject *nurse, PyObject *patient) { auto &internals = get_internals(); auto instance = reinterpret_cast(nurse); - auto ¤t_patients = internals.patients[nurse]; instance->has_patients = true; - for (auto &p : current_patients) - if (p == patient) - return; Py_INCREF(patient); - current_patients.push_back(patient); + internals.patients[nurse].push_back(patient); } inline void clear_patients(PyObject *self) { @@ -472,7 +469,7 @@ if (tinfo && tinfo->get_buffer) break; } - if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) { + if (view == nullptr || !tinfo || !tinfo->get_buffer) { if (view) view->obj = nullptr; PyErr_SetString(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); @@ -589,6 +586,9 @@ type->tp_as_number = &heap_type->as_number; type->tp_as_sequence = &heap_type->as_sequence; type->tp_as_mapping = &heap_type->as_mapping; +#if PY_VERSION_HEX >= 0x03050000 + type->tp_as_async = &heap_type->as_async; +#endif /* Flags */ type->tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/common.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/common.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/common.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/common.h 2019-10-11 09:23:26.000000000 +0000 @@ -93,8 +93,8 @@ #endif #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 2 -#define PYBIND11_VERSION_PATCH 4 +#define PYBIND11_VERSION_MINOR 4 +#define PYBIND11_VERSION_PATCH 2 /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) @@ -113,10 +113,6 @@ #include #include -#if defined(_WIN32) && (defined(min) || defined(max)) -# error Macro clash with min and max -- define NOMINMAX when compiling your program on Windows -#endif - #if defined(isalnum) # undef isalnum # undef isalpha @@ -159,6 +155,8 @@ #define PYBIND11_BYTES_SIZE PyBytes_Size #define PYBIND11_LONG_CHECK(o) PyLong_Check(o) #define PYBIND11_LONG_AS_LONGLONG(o) PyLong_AsLongLong(o) +#define PYBIND11_LONG_FROM_SIGNED(o) PyLong_FromSsize_t((ssize_t) o) +#define PYBIND11_LONG_FROM_UNSIGNED(o) PyLong_FromSize_t((size_t) o) #define PYBIND11_BYTES_NAME "bytes" #define PYBIND11_STRING_NAME "str" #define PYBIND11_SLICE_OBJECT PyObject @@ -166,7 +164,9 @@ #define PYBIND11_STR_TYPE ::pybind11::str #define PYBIND11_BOOL_ATTR "__bool__" #define PYBIND11_NB_BOOL(ptr) ((ptr)->nb_bool) +// Providing a separate declaration to make Clang's -Wmissing-prototypes happy #define PYBIND11_PLUGIN_IMPL(name) \ + extern "C" PYBIND11_EXPORT PyObject *PyInit_##name(); \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() #else @@ -181,6 +181,8 @@ #define PYBIND11_BYTES_SIZE PyString_Size #define PYBIND11_LONG_CHECK(o) (PyInt_Check(o) || PyLong_Check(o)) #define PYBIND11_LONG_AS_LONGLONG(o) (PyInt_Check(o) ? (long long) PyLong_AsLong(o) : PyLong_AsLongLong(o)) +#define PYBIND11_LONG_FROM_SIGNED(o) PyInt_FromSsize_t((ssize_t) o) // Returns long if needed. +#define PYBIND11_LONG_FROM_UNSIGNED(o) PyInt_FromSize_t((size_t) o) // Returns long if needed. #define PYBIND11_BYTES_NAME "str" #define PYBIND11_STRING_NAME "unicode" #define PYBIND11_SLICE_OBJECT PySliceObject @@ -188,8 +190,10 @@ #define PYBIND11_STR_TYPE ::pybind11::bytes #define PYBIND11_BOOL_ATTR "__nonzero__" #define PYBIND11_NB_BOOL(ptr) ((ptr)->nb_nonzero) +// Providing a separate PyInit decl to make Clang's -Wmissing-prototypes happy #define PYBIND11_PLUGIN_IMPL(name) \ static PyObject *pybind11_init_wrapper(); \ + extern "C" PYBIND11_EXPORT void init##name(); \ extern "C" PYBIND11_EXPORT void init##name() { \ (void)pybind11_init_wrapper(); \ } \ @@ -208,6 +212,31 @@ #define PYBIND11_TOSTRING(x) PYBIND11_STRINGIFY(x) #define PYBIND11_CONCAT(first, second) first##second +#define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *compiled_ver = PYBIND11_TOSTRING(PY_MAJOR_VERSION) \ + "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ + const char *runtime_ver = Py_GetVersion(); \ + size_t len = std::strlen(compiled_ver); \ + if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ + || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for Python %s, " \ + "but the interpreter version is incompatible: %s.", \ + compiled_ver, runtime_ver); \ + return nullptr; \ + } \ + } + +#define PYBIND11_CATCH_INIT_EXCEPTIONS \ + catch (pybind11::error_already_set &e) { \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } catch (const std::exception &e) { \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } \ + /** \rst ***Deprecated in favor of PYBIND11_MODULE*** @@ -227,27 +256,10 @@ PYBIND11_DEPRECATED("PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE") \ static PyObject *pybind11_init(); \ PYBIND11_PLUGIN_IMPL(name) { \ - int major, minor; \ - if (sscanf(Py_GetVersion(), "%i.%i", &major, &minor) != 2) { \ - PyErr_SetString(PyExc_ImportError, "Can't parse Python version."); \ - return nullptr; \ - } else if (major != PY_MAJOR_VERSION || minor != PY_MINOR_VERSION) { \ - PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for " \ - "version %i.%i, while the interpreter is running " \ - "version %i.%i.", PY_MAJOR_VERSION, PY_MINOR_VERSION, \ - major, minor); \ - return nullptr; \ - } \ + PYBIND11_CHECK_PYTHON_VERSION \ try { \ return pybind11_init(); \ - } catch (pybind11::error_already_set &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } \ + } PYBIND11_CATCH_INIT_EXCEPTIONS \ } \ PyObject *pybind11_init() @@ -271,29 +283,12 @@ #define PYBIND11_MODULE(name, variable) \ static void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &); \ PYBIND11_PLUGIN_IMPL(name) { \ - int major, minor; \ - if (sscanf(Py_GetVersion(), "%i.%i", &major, &minor) != 2) { \ - PyErr_SetString(PyExc_ImportError, "Can't parse Python version."); \ - return nullptr; \ - } else if (major != PY_MAJOR_VERSION || minor != PY_MINOR_VERSION) { \ - PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for " \ - "version %i.%i, while the interpreter is running " \ - "version %i.%i.", PY_MAJOR_VERSION, PY_MINOR_VERSION, \ - major, minor); \ - return nullptr; \ - } \ + PYBIND11_CHECK_PYTHON_VERSION \ auto m = pybind11::module(PYBIND11_TOSTRING(name)); \ try { \ PYBIND11_CONCAT(pybind11_init_, name)(m); \ return m.ptr(); \ - } catch (pybind11::error_already_set &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } \ + } PYBIND11_CATCH_INIT_EXCEPTIONS \ } \ void PYBIND11_CONCAT(pybind11_init_, name)(pybind11::module &variable) @@ -391,7 +386,7 @@ void *simple_value_holder[1 + instance_simple_holder_in_ptrs()]; nonsimple_values_and_holders nonsimple; }; - /// Weak references (needed for keep alive): + /// Weak references PyObject *weakrefs; /// If true, the pointer is owned which means we're free to manage it with a holder. bool owned : 1; @@ -408,10 +403,10 @@ * (which is typically the size of two pointers), or when multiple inheritance is used on the * python side. Non-simple layout allocates the required amount of memory to have multiple * bound C++ classes as parents. Under this layout, `nonsimple.values_and_holders` is set to a - * pointer to allocated space of the required space to hold a a sequence of value pointers and + * pointer to allocated space of the required space to hold a sequence of value pointers and * holders followed `status`, a set of bit flags (1 byte each), i.e. * [val1*][holder1][val2*][holder2]...[bb...] where each [block] is rounded up to a multiple of - * `sizeof(void *)`. `nonsimple.holder_constructed` is, for convenience, a pointer to the + * `sizeof(void *)`. `nonsimple.status` is, for convenience, a pointer to the * beginning of the [bb...] block (but not independently allocated). * * Status bits indicate whether the associated holder is constructed (& @@ -584,6 +579,11 @@ template using is_strict_base_of = bool_constant< std::is_base_of::value && !std::is_same::value>; +/// Like is_base_of, but also requires that the base type is accessible (i.e. that a Derived pointer +/// can be converted to a Base pointer) +template using is_accessible_base_of = bool_constant< + std::is_base_of::value && std::is_convertible::value>; + template class Base> struct is_template_base_of_impl { template static std::true_type check(Base *); @@ -673,6 +673,7 @@ PYBIND11_RUNTIME_EXCEPTION(key_error, PyExc_KeyError) PYBIND11_RUNTIME_EXCEPTION(value_error, PyExc_ValueError) PYBIND11_RUNTIME_EXCEPTION(type_error, PyExc_TypeError) +PYBIND11_RUNTIME_EXCEPTION(buffer_error, PyExc_BufferError) PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybind11::cast or handle::call fail due to a type casting error PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally @@ -702,9 +703,13 @@ static std::string format() { return std::string(1, c); } }; +#if !defined(PYBIND11_CPP17) + template constexpr const char format_descriptor< T, detail::enable_if_t::value>>::value[2]; +#endif + /// RAII wrapper that temporarily clears any Python error state struct error_scope { PyObject *type, *value, *trace; @@ -715,10 +720,6 @@ /// Dummy destructor wrapper that can be used to expose classes with a private destructor struct nodelete { template void operator()(T*) { } }; -// overload_cast requires variable templates: C++14 -#if defined(PYBIND11_CPP14) -#define PYBIND11_OVERLOAD_CAST 1 - NAMESPACE_BEGIN(detail) template struct overload_cast_impl { @@ -738,19 +739,23 @@ }; NAMESPACE_END(detail) +// overload_cast requires variable templates: C++14 +#if defined(PYBIND11_CPP14) +#define PYBIND11_OVERLOAD_CAST 1 /// Syntax sugar for resolving overloaded function pointers: /// - regular: static_cast(&Class::func) /// - sweet: overload_cast(&Class::func) template static constexpr detail::overload_cast_impl overload_cast = {}; // MSVC 2015 only accepts this particular initialization syntax for this variable template. +#endif /// Const member function selector for overload_cast /// - regular: static_cast(&Class::func) /// - sweet: overload_cast(&Class::func, const_) static constexpr auto const_ = std::true_type{}; -#else // no overload_cast: providing something that static_assert-fails: +#if !defined(PYBIND11_CPP14) // no overload_cast: providing something that static_assert-fails: template struct overload_cast { static_assert(detail::deferred_t::value, "pybind11::overload_cast<...> requires compiling in C++14 mode"); diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/descr.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/descr.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/descr.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/descr.h 2019-10-08 08:35:40.000000000 +0000 @@ -1,6 +1,5 @@ /* - pybind11/detail/descr.h: Helper type for concatenating type signatures - either at runtime (C++11) or compile time (C++14) + pybind11/detail/descr.h: Helper type for concatenating type signatures at compile time Copyright (c) 2016 Wenzel Jakob @@ -15,171 +14,87 @@ NAMESPACE_BEGIN(PYBIND11_NAMESPACE) NAMESPACE_BEGIN(detail) -/* Concatenate type signatures at compile time using C++14 */ -#if defined(PYBIND11_CPP14) && !defined(_MSC_VER) -#define PYBIND11_CONSTEXPR_DESCR - -template class descr { - template friend class descr; -public: - constexpr descr(char const (&text) [Size1+1], const std::type_info * const (&types)[Size2+1]) - : descr(text, types, - make_index_sequence(), - make_index_sequence()) { } - - constexpr const char *text() const { return m_text; } - constexpr const std::type_info * const * types() const { return m_types; } - - template - constexpr descr operator+(const descr &other) const { - return concat(other, - make_index_sequence(), - make_index_sequence(), - make_index_sequence(), - make_index_sequence()); - } +#if !defined(_MSC_VER) +# define PYBIND11_DESCR_CONSTEXPR static constexpr +#else +# define PYBIND11_DESCR_CONSTEXPR const +#endif -protected: - template - constexpr descr( - char const (&text) [Size1+1], - const std::type_info * const (&types) [Size2+1], - index_sequence, index_sequence) - : m_text{text[Indices1]..., '\0'}, - m_types{types[Indices2]..., nullptr } {} - - template - constexpr descr - concat(const descr &other, - index_sequence, index_sequence, - index_sequence, index_sequence) const { - return descr( - { m_text[Indices1]..., other.m_text[OtherIndices1]..., '\0' }, - { m_types[Indices2]..., other.m_types[OtherIndices2]..., nullptr } - ); - } +/* Concatenate type signatures at compile time */ +template +struct descr { + char text[N + 1]; + + constexpr descr() : text{'\0'} { } + constexpr descr(char const (&s)[N+1]) : descr(s, make_index_sequence()) { } + + template + constexpr descr(char const (&s)[N+1], index_sequence) : text{s[Is]..., '\0'} { } -protected: - char m_text[Size1 + 1]; - const std::type_info * m_types[Size2 + 1]; + template + constexpr descr(char c, Chars... cs) : text{c, static_cast(cs)..., '\0'} { } + + static constexpr std::array types() { + return {{&typeid(Ts)..., nullptr}}; + } }; -template constexpr descr _(char const(&text)[Size]) { - return descr(text, { nullptr }); +template +constexpr descr plus_impl(const descr &a, const descr &b, + index_sequence, index_sequence) { + return {a.text[Is1]..., b.text[Is2]...}; +} + +template +constexpr descr operator+(const descr &a, const descr &b) { + return plus_impl(a, b, make_index_sequence(), make_index_sequence()); } +template +constexpr descr _(char const(&text)[N]) { return descr(text); } +constexpr descr<0> _(char const(&)[1]) { return {}; } + template struct int_to_str : int_to_str { }; template struct int_to_str<0, Digits...> { - static constexpr auto digits = descr({ ('0' + Digits)..., '\0' }, { nullptr }); + static constexpr auto digits = descr(('0' + Digits)...); }; // Ternary description (like std::conditional) -template -constexpr enable_if_t> _(char const(&text1)[Size1], char const(&)[Size2]) { +template +constexpr enable_if_t> _(char const(&text1)[N1], char const(&)[N2]) { return _(text1); } -template -constexpr enable_if_t> _(char const(&)[Size1], char const(&text2)[Size2]) { +template +constexpr enable_if_t> _(char const(&)[N1], char const(&text2)[N2]) { return _(text2); } -template -constexpr enable_if_t> _(descr d, descr) { return d; } -template -constexpr enable_if_t> _(descr, descr d) { return d; } + +template +constexpr enable_if_t _(const T1 &d, const T2 &) { return d; } +template +constexpr enable_if_t _(const T1 &, const T2 &d) { return d; } template auto constexpr _() -> decltype(int_to_str::digits) { return int_to_str::digits; } -template constexpr descr<1, 1> _() { - return descr<1, 1>({ '%', '\0' }, { &typeid(Type), nullptr }); -} - -inline constexpr descr<0, 0> concat() { return _(""); } -template auto constexpr concat(descr descr) { return descr; } -template auto constexpr concat(descr descr, Args&&... args) { return descr + _(", ") + concat(args...); } -template auto constexpr type_descr(descr descr) { return _("{") + descr + _("}"); } - -#define PYBIND11_DESCR constexpr auto - -#else /* Simpler C++11 implementation based on run-time memory allocation and copying */ - -class descr { -public: - PYBIND11_NOINLINE descr(const char *text, const std::type_info * const * types) { - size_t nChars = len(text), nTypes = len(types); - m_text = new char[nChars]; - m_types = new const std::type_info *[nTypes]; - memcpy(m_text, text, nChars * sizeof(char)); - memcpy(m_types, types, nTypes * sizeof(const std::type_info *)); - } - - PYBIND11_NOINLINE descr operator+(descr &&d2) && { - descr r; - - size_t nChars1 = len(m_text), nTypes1 = len(m_types); - size_t nChars2 = len(d2.m_text), nTypes2 = len(d2.m_types); +template constexpr descr<1, Type> _() { return {'%'}; } - r.m_text = new char[nChars1 + nChars2 - 1]; - r.m_types = new const std::type_info *[nTypes1 + nTypes2 - 1]; - memcpy(r.m_text, m_text, (nChars1-1) * sizeof(char)); - memcpy(r.m_text + nChars1 - 1, d2.m_text, nChars2 * sizeof(char)); - memcpy(r.m_types, m_types, (nTypes1-1) * sizeof(std::type_info *)); - memcpy(r.m_types + nTypes1 - 1, d2.m_types, nTypes2 * sizeof(std::type_info *)); +constexpr descr<0> concat() { return {}; } - delete[] m_text; delete[] m_types; - delete[] d2.m_text; delete[] d2.m_types; +template +constexpr descr concat(const descr &descr) { return descr; } - return r; - } - - char *text() { return m_text; } - const std::type_info * * types() { return m_types; } - -protected: - PYBIND11_NOINLINE descr() { } - - template static size_t len(const T *ptr) { // return length including null termination - const T *it = ptr; - while (*it++ != (T) 0) - ; - return static_cast(it - ptr); - } - - const std::type_info **m_types = nullptr; - char *m_text = nullptr; -}; - -/* The 'PYBIND11_NOINLINE inline' combinations below are intentional to get the desired linkage while producing as little object code as possible */ - -PYBIND11_NOINLINE inline descr _(const char *text) { - const std::type_info *types[1] = { nullptr }; - return descr(text, types); -} - -template PYBIND11_NOINLINE enable_if_t _(const char *text1, const char *) { return _(text1); } -template PYBIND11_NOINLINE enable_if_t _(char const *, const char *text2) { return _(text2); } -template PYBIND11_NOINLINE enable_if_t _(descr d, descr) { return d; } -template PYBIND11_NOINLINE enable_if_t _(descr, descr d) { return d; } - -template PYBIND11_NOINLINE descr _() { - const std::type_info *types[2] = { &typeid(Type), nullptr }; - return descr("%", types); +template +constexpr auto concat(const descr &d, const Args &...args) + -> decltype(std::declval>() + concat(args...)) { + return d + _(", ") + concat(args...); } -template PYBIND11_NOINLINE descr _() { - const std::type_info *types[1] = { nullptr }; - return descr(std::to_string(Size).c_str(), types); +template +constexpr descr type_descr(const descr &descr) { + return _("{") + descr + _("}"); } -PYBIND11_NOINLINE inline descr concat() { return _(""); } -PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; } -template PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward(args)...); } -PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); } - -#define PYBIND11_DESCR ::pybind11::detail::descr -#endif - NAMESPACE_END(detail) NAMESPACE_END(PYBIND11_NAMESPACE) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/init.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/init.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/init.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/init.h 2019-10-08 08:35:40.000000000 +0000 @@ -24,7 +24,7 @@ template using cast_op_type = value_and_holder &; operator value_and_holder &() { return *value; } - static PYBIND11_DESCR name() { return type_descr(_()); } + static constexpr auto name = _(); private: value_and_holder *value = nullptr; diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/internals.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/internals.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/internals.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/internals.h 2019-10-08 08:35:40.000000000 +0000 @@ -23,7 +23,7 @@ #if PY_VERSION_HEX >= 0x03070000 # define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr # define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (tstate)) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) # define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) #else // Usually an int but a long on Cygwin64 with Python 3.x @@ -116,7 +116,7 @@ struct type_info { PyTypeObject *type; const std::type_info *cpptype; - size_t type_size, holder_size_in_ptrs; + size_t type_size, type_align, holder_size_in_ptrs; void *(*operator_new)(size_t); void (*init_instance)(instance *, const void *); void (*dealloc)(value_and_holder &v_h); @@ -138,7 +138,48 @@ }; /// Tracks the `internals` and `type_info` ABI version independent of the main library version -#define PYBIND11_INTERNALS_VERSION 2 +#define PYBIND11_INTERNALS_VERSION 3 + +/// On MSVC, debug and release builds are not ABI-compatible! +#if defined(_MSC_VER) && defined(_DEBUG) +# define PYBIND11_BUILD_TYPE "_debug" +#else +# define PYBIND11_BUILD_TYPE "" +#endif + +/// Let's assume that different compilers are ABI-incompatible. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "_msvc" +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_TYPE "_icc" +#elif defined(__clang__) +# define PYBIND11_COMPILER_TYPE "_clang" +#elif defined(__PGI) +# define PYBIND11_COMPILER_TYPE "_pgi" +#elif defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "_mingw" +#elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "_gcc" +#else +# define PYBIND11_COMPILER_TYPE "_unknown" +#endif + +#if defined(_LIBCPP_VERSION) +# define PYBIND11_STDLIB "_libcpp" +#elif defined(__GLIBCXX__) || defined(__GLIBCPP__) +# define PYBIND11_STDLIB "_libstdcpp" +#else +# define PYBIND11_STDLIB "" +#endif + +/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. +#if defined(__GXX_ABI_VERSION) +# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +#else +# define PYBIND11_BUILD_ABI "" +#endif #if defined(WITH_THREAD) # define PYBIND11_INTERNALS_KIND "" @@ -147,10 +188,10 @@ #endif #define PYBIND11_INTERNALS_ID "__pybind11_internals_v" \ - PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND "__" + PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" #define PYBIND11_MODULE_LOCAL_ID "__pybind11_module_local_v" \ - PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND "__" + PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" /// Each module locally stores a pointer to the `internals` data. The data /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. @@ -159,12 +200,48 @@ return internals_pp; } +inline void translate_exception(std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (error_already_set &e) { e.restore(); return; + } catch (const builtin_exception &e) { e.set_error(); return; + } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; + } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::length_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::out_of_range &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; + } catch (const std::range_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::exception &e) { PyErr_SetString(PyExc_RuntimeError, e.what()); return; + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!"); + return; + } +} + +#if !defined(__GLIBCXX__) +inline void translate_local_exception(std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (error_already_set &e) { e.restore(); return; + } catch (const builtin_exception &e) { e.set_error(); return; + } +} +#endif + /// Return a reference to the current `internals` data PYBIND11_NOINLINE inline internals &get_internals() { auto **&internals_pp = get_internals_pp(); if (internals_pp && *internals_pp) return **internals_pp; + // Ensure that the GIL is held since we will need to make Python calls. + // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. + struct gil_scoped_acquire_local { + gil_scoped_acquire_local() : state (PyGILState_Ensure()) {} + ~gil_scoped_acquire_local() { PyGILState_Release(state); } + const PyGILState_STATE state; + } gil; + constexpr auto *id = PYBIND11_INTERNALS_ID; auto builtins = handle(PyEval_GetBuiltins()); if (builtins.contains(id) && isinstance(builtins[id])) { @@ -176,15 +253,7 @@ // // libstdc++ doesn't require this (types there are identified only by name) #if !defined(__GLIBCXX__) - (*internals_pp)->registered_exception_translators.push_front( - [](std::exception_ptr p) -> void { - try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; - } - } - ); + (*internals_pp)->registered_exception_translators.push_front(&translate_local_exception); #endif } else { if (!internals_pp) internals_pp = new internals*(); @@ -207,25 +276,7 @@ internals_ptr->istate = tstate->interp; #endif builtins[id] = capsule(internals_pp); - internals_ptr->registered_exception_translators.push_front( - [](std::exception_ptr p) -> void { - try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; - } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; - } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::length_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::out_of_range &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; - } catch (const std::range_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::exception &e) { PyErr_SetString(PyExc_RuntimeError, e.what()); return; - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!"); - return; - } - } - ); + internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); internals_ptr->instance_base = make_object_base_type(internals_ptr->default_metaclass); diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/typeid.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/typeid.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/detail/typeid.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/detail/typeid.h 2019-10-08 08:35:40.000000000 +0000 @@ -16,6 +16,8 @@ #include #endif +#include "common.h" + NAMESPACE_BEGIN(PYBIND11_NAMESPACE) NAMESPACE_BEGIN(detail) /// Erase all occurrences of a substring diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/eigen.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/eigen.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/eigen.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/eigen.h 2019-10-08 08:35:40.000000000 +0000 @@ -17,6 +17,11 @@ # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wconversion" # pragma GCC diagnostic ignored "-Wdeprecated-declarations" +# ifdef __clang__ +// Eigen generates a bunch of implicit-copy-constructor-is-deprecated warnings with -Wdeprecated +// under Clang, so disable that warning here: +# pragma GCC diagnostic ignored "-Wdeprecated" +# endif # if __GNUC__ >= 7 # pragma GCC diagnostic ignored "-Wint-in-bool-context" # endif @@ -181,28 +186,26 @@ } } - static PYBIND11_DESCR descriptor() { - constexpr bool show_writeable = is_eigen_dense_map::value && is_eigen_mutable_map::value; - constexpr bool show_order = is_eigen_dense_map::value; - constexpr bool show_c_contiguous = show_order && requires_row_major; - constexpr bool show_f_contiguous = !show_c_contiguous && show_order && requires_col_major; - - return type_descr(_("numpy.ndarray[") + npy_format_descriptor::name() + - _("[") + _(_<(size_t) rows>(), _("m")) + - _(", ") + _(_<(size_t) cols>(), _("n")) + - _("]") + - // For a reference type (e.g. Ref) we have other constraints that might need to be - // satisfied: writeable=True (for a mutable reference), and, depending on the map's stride - // options, possibly f_contiguous or c_contiguous. We include them in the descriptor output - // to provide some hint as to why a TypeError is occurring (otherwise it can be confusing to - // see that a function accepts a 'numpy.ndarray[float64[3,2]]' and an error message that you - // *gave* a numpy.ndarray of the right type and dimensions. - _(", flags.writeable", "") + - _(", flags.c_contiguous", "") + - _(", flags.f_contiguous", "") + - _("]") - ); - } + static constexpr bool show_writeable = is_eigen_dense_map::value && is_eigen_mutable_map::value; + static constexpr bool show_order = is_eigen_dense_map::value; + static constexpr bool show_c_contiguous = show_order && requires_row_major; + static constexpr bool show_f_contiguous = !show_c_contiguous && show_order && requires_col_major; + + static constexpr auto descriptor = + _("numpy.ndarray[") + npy_format_descriptor::name + + _("[") + _(_<(size_t) rows>(), _("m")) + + _(", ") + _(_<(size_t) cols>(), _("n")) + + _("]") + + // For a reference type (e.g. Ref) we have other constraints that might need to be + // satisfied: writeable=True (for a mutable reference), and, depending on the map's stride + // options, possibly f_contiguous or c_contiguous. We include them in the descriptor output + // to provide some hint as to why a TypeError is occurring (otherwise it can be confusing to + // see that a function accepts a 'numpy.ndarray[float64[3,2]]' and an error message that you + // *gave* a numpy.ndarray of the right type and dimensions. + _(", flags.writeable", "") + + _(", flags.c_contiguous", "") + + _(", flags.f_contiguous", "") + + _("]"); }; // Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, @@ -339,7 +342,7 @@ return cast_impl(src, policy, parent); } - static PYBIND11_DESCR name() { return props::descriptor(); } + static constexpr auto name = props::descriptor; operator Type*() { return &value; } operator Type&() { return value; } @@ -379,7 +382,7 @@ } } - static PYBIND11_DESCR name() { return props::descriptor(); } + static constexpr auto name = props::descriptor; // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return // types but not bound arguments). We still provide them (with an explicitly delete) so that @@ -524,7 +527,7 @@ } static handle cast(const Type *src, return_value_policy policy, handle parent) { return cast(*src, policy, parent); } - static PYBIND11_DESCR name() { return props::descriptor(); } + static constexpr auto name = props::descriptor; // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return // types but not bound arguments). We still provide them (with an explicitly delete) so that @@ -591,7 +594,7 @@ } PYBIND11_TYPE_CASTER(Type, _<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[") - + npy_format_descriptor::name() + _("]")); + + npy_format_descriptor::name + _("]")); }; NAMESPACE_END(detail) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/embed.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/embed.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/embed.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/embed.h 2019-10-08 08:35:40.000000000 +0000 @@ -90,8 +90,14 @@ Initialize the Python interpreter. No other pybind11 or CPython API functions can be called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The optional parameter can be used to skip the registration of signal handlers (see the - Python documentation for details). Calling this function again after the interpreter + `Python documentation`_ for details). Calling this function again after the interpreter has already been initialized is a fatal error. + + If initializing the Python interpreter fails, then the program is terminated. (This + is controlled by the CPython runtime and is an exception to pybind11's normal behavior + of throwing exceptions on errors.) + + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx \endrst */ inline void initialize_interpreter(bool init_signal_handlers = true) { if (Py_IsInitialized()) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/functional.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/functional.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/functional.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/functional.h 2019-10-08 08:35:40.000000000 +0000 @@ -54,12 +54,30 @@ } } - value = [func](Args... args) -> Return { - gil_scoped_acquire acq; - object retval(func(std::forward(args)...)); - /* Visual studio 2015 parser issue: need parentheses around this expression */ - return (retval.template cast()); + // ensure GIL is held during functor destruction + struct func_handle { + function f; + func_handle(function&& f_) : f(std::move(f_)) {} + func_handle(const func_handle&) = default; + ~func_handle() { + gil_scoped_acquire acq; + function kill_f(std::move(f)); + } }; + + // to emulate 'move initialization capture' in C++11 + struct func_wrapper { + func_handle hfunc; + func_wrapper(func_handle&& hf): hfunc(std::move(hf)) {} + Return operator()(Args... args) const { + gil_scoped_acquire acq; + object retval(hfunc.f(std::forward(args)...)); + /* Visual studio 2015 parser issue: need parentheses around this expression */ + return (retval.template cast()); + } + }; + + value = func_wrapper(func_handle(std::move(func))); return true; } @@ -75,10 +93,8 @@ return cpp_function(std::forward(f_), policy).release(); } - PYBIND11_TYPE_CASTER(type, _("Callable[[") + - argument_loader::arg_names() + _("], ") + - make_caster::name() + - _("]")); + PYBIND11_TYPE_CASTER(type, _("Callable[[") + concat(make_caster::name...) + _("], ") + + make_caster::name + _("]")); }; NAMESPACE_END(detail) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/iostream.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/iostream.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/iostream.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/iostream.h 2019-10-08 08:35:40.000000000 +0000 @@ -25,7 +25,8 @@ private: using traits_type = std::streambuf::traits_type; - char d_buffer[1024]; + const size_t buf_size; + std::unique_ptr d_buffer; object pywrite; object pyflush; @@ -42,8 +43,11 @@ // This subtraction cannot be negative, so dropping the sign str line(pbase(), static_cast(pptr() - pbase())); - pywrite(line); - pyflush(); + { + gil_scoped_acquire tmp; + pywrite(line); + pyflush(); + } setp(pbase(), epptr()); } @@ -51,12 +55,17 @@ } public: - pythonbuf(object pyostream) - : pywrite(pyostream.attr("write")), + + pythonbuf(object pyostream, size_t buffer_size = 1024) + : buf_size(buffer_size), + d_buffer(new char[buf_size]), + pywrite(pyostream.attr("write")), pyflush(pyostream.attr("flush")) { - setp(d_buffer, d_buffer + sizeof(d_buffer) - 1); + setp(d_buffer.get(), d_buffer.get() + buf_size - 1); } + pythonbuf(pythonbuf&&) = default; + /// Sync before destroy ~pythonbuf() { sync(); @@ -194,7 +203,7 @@ return class_(m, name.c_str(), module_local()) .def(init(), arg("stdout")=true, arg("stderr")=true) .def("__enter__", &detail::OstreamRedirect::enter) - .def("__exit__", [](detail::OstreamRedirect &self, args) { self.exit(); }); + .def("__exit__", [](detail::OstreamRedirect &self_, args) { self_.exit(); }); } NAMESPACE_END(PYBIND11_NAMESPACE) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/numpy.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/numpy.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/numpy.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/numpy.h 2019-10-11 09:23:26.000000000 +0000 @@ -14,13 +14,14 @@ #include #include #include +#include #include #include #include #include -#include #include #include +#include #include #if defined(_MSC_VER) @@ -108,6 +109,18 @@ return *ptr; } +template struct same_size { + template using as = bool_constant; +}; + +template constexpr int platform_lookup() { return -1; } + +// Lookup a type according to its size, and return a value corresponding to the NumPy typenum. +template +constexpr int platform_lookup(int I, Ints... Is) { + return sizeof(Concrete) == sizeof(T) ? I : platform_lookup(Is...); +} + struct npy_api { enum constants { NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, @@ -126,7 +139,23 @@ NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, NPY_OBJECT_ = 17, - NPY_STRING_, NPY_UNICODE_, NPY_VOID_ + NPY_STRING_, NPY_UNICODE_, NPY_VOID_, + // Platform-dependent normalization + NPY_INT8_ = NPY_BYTE_, + NPY_UINT8_ = NPY_UBYTE_, + NPY_INT16_ = NPY_SHORT_, + NPY_UINT16_ = NPY_USHORT_, + // `npy_common.h` defines the integer aliases. In order, it checks: + // NPY_BITSOF_LONG, NPY_BITSOF_LONGLONG, NPY_BITSOF_INT, NPY_BITSOF_SHORT, NPY_BITSOF_CHAR + // and assigns the alias to the first matching size, so we should check in this order. + NPY_INT32_ = platform_lookup( + NPY_LONG_, NPY_INT_, NPY_SHORT_), + NPY_UINT32_ = platform_lookup( + NPY_ULONG_, NPY_UINT_, NPY_USHORT_), + NPY_INT64_ = platform_lookup( + NPY_LONG_, NPY_LONGLONG_, NPY_INT_), + NPY_UINT64_ = platform_lookup( + NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_), }; typedef struct { @@ -250,7 +279,7 @@ typedef T type; static constexpr bool is_array = false; static constexpr bool is_empty = false; - static PYBIND11_DESCR extents() { return _(""); } + static constexpr auto extents = _(""); static void append_extents(list& /* shape */) { } }; // Computes underlying type and a comma-separated list of extents for array @@ -269,15 +298,9 @@ array_info::append_extents(shape); } - template::is_array, int> = 0> - static PYBIND11_DESCR extents() { - return _(); - } - - template::is_array, int> = 0> - static PYBIND11_DESCR extents() { - return concat(_(), array_info::extents()); - } + static constexpr auto extents = _::is_array>( + concat(_(), array_info::extents), _() + ); }; // For numpy we have special handling for arrays of characters, so we don't include // the size in the array extents. @@ -446,7 +469,7 @@ /// This is essentially the same as calling numpy.dtype(args) in Python. static dtype from_args(object args) { PyObject *ptr = nullptr; - if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) + if (!detail::npy_api::get().PyArray_DescrConverter_(args.ptr(), &ptr) || !ptr) throw error_already_set(); return reinterpret_steal(ptr); } @@ -861,14 +884,14 @@ // Reference to element at a given index template const T& at(Ix... index) const { - if (sizeof...(index) != ndim()) + if ((ssize_t) sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); return *(static_cast(array::data()) + byte_offset(ssize_t(index)...) / itemsize()); } // Mutable reference to element at a given index template T& mutable_at(Ix... index) { - if (sizeof...(index) != ndim()) + if ((ssize_t) sizeof...(index) != ndim()) fail_dim_check(sizeof...(index), "index dimension mismatch"); return *(static_cast(array::mutable_data()) + byte_offset(ssize_t(index)...) / itemsize()); } @@ -948,8 +971,8 @@ struct format_descriptor::is_array>> { static std::string format() { using namespace detail; - PYBIND11_DESCR extents = _("(") + array_info::extents() + _(")"); - return extents.text() + format_descriptor>::format(); + static constexpr auto extents = _("(") + array_info::extents + _(")"); + return extents.text + format_descriptor>::format(); } }; @@ -968,7 +991,7 @@ static handle cast(const handle &src, return_value_policy /* policy */, handle /* parent */) { return src.inc_ref(); } - PYBIND11_TYPE_CASTER(type, handle_type_name::name()); + PYBIND11_TYPE_CASTER(type, handle_type_name::name); }; template @@ -978,13 +1001,40 @@ } }; -template struct npy_format_descriptor::value>> { +template +struct npy_format_descriptor_name; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value>( + _("bool"), _::value>("int", "uint") + _() + ); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value || std::is_same::value>( + _("float") + _(), _("longdouble") + ); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = _::value + || std::is_same::value>( + _("complex") + _(), _("longcomplex") + ); +}; + +template +struct npy_format_descriptor::value>> + : npy_format_descriptor_name { private: // NB: the order here must match the one in common.h constexpr static const int values[15] = { npy_api::NPY_BOOL_, - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, - npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_, + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_INT16_, npy_api::NPY_UINT16_, + npy_api::NPY_INT32_, npy_api::NPY_UINT32_, npy_api::NPY_INT64_, npy_api::NPY_UINT64_, npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ }; @@ -994,28 +1044,13 @@ static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) - return reinterpret_borrow(ptr); + return reinterpret_steal(ptr); pybind11_fail("Unsupported buffer format!"); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value>(_("bool"), - _::value>("int", "uint") + _()); - } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value || std::is_same::value>( - _("float") + _(), _("longdouble")); - } - template ::value, int> = 0> - static PYBIND11_DESCR name() { - return _::value || std::is_same::value>( - _("complex") + _(), _("longcomplex")); - } }; #define PYBIND11_DECL_CHAR_FMT \ - static PYBIND11_DESCR name() { return _("S") + _(); } \ + static constexpr auto name = _("S") + _(); \ static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } template struct npy_format_descriptor { PYBIND11_DECL_CHAR_FMT }; template struct npy_format_descriptor> { PYBIND11_DECL_CHAR_FMT }; @@ -1027,7 +1062,7 @@ public: static_assert(!array_info::is_empty, "Zero-sized arrays are not supported"); - static PYBIND11_DESCR name() { return _("(") + array_info::extents() + _(")") + base_descr::name(); } + static constexpr auto name = _("(") + array_info::extents + _(")") + base_descr::name; static pybind11::dtype dtype() { list shape; array_info::append_extents(shape); @@ -1039,7 +1074,7 @@ private: using base_descr = npy_format_descriptor::type>; public: - static PYBIND11_DESCR name() { return base_descr::name(); } + static constexpr auto name = base_descr::name; static pybind11::dtype dtype() { return base_descr::dtype(); } }; @@ -1052,7 +1087,7 @@ }; inline PYBIND11_NOINLINE void register_structured_dtype( - const std::initializer_list& fields, + any_container fields, const std::type_info& tinfo, ssize_t itemsize, bool (*direct_converter)(PyObject *, void *&)) { @@ -1060,8 +1095,14 @@ if (numpy_internals.get_type_info(tinfo, false)) pybind11_fail("NumPy: dtype is already registered"); + // Use ordered fields because order matters as of NumPy 1.14: + // https://docs.scipy.org/doc/numpy/release.html#multiple-field-indexing-assignment-of-structured-arrays + std::vector ordered_fields(std::move(fields)); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); + list names, formats, offsets; - for (auto field : fields) { + for (auto& field : ordered_fields) { if (!field.descr) pybind11_fail(std::string("NumPy: unsupported field dtype: `") + field.name + "` @ " + tinfo.name()); @@ -1078,9 +1119,6 @@ // - https://github.com/numpy/numpy/pull/7798 // Because of this, we won't use numpy's logic to generate buffer format // strings and will just do it ourselves. - std::vector ordered_fields(fields); - std::sort(ordered_fields.begin(), ordered_fields.end(), - [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); ssize_t offset = 0; std::ostringstream oss; // mark the structure as unaligned with '^', because numpy and C++ don't @@ -1114,7 +1152,7 @@ template struct npy_format_descriptor { static_assert(is_pod_struct::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); - static PYBIND11_DESCR name() { return make_caster::name(); } + static constexpr auto name = make_caster::name; static pybind11::dtype dtype() { return reinterpret_borrow(dtype_ptr()); @@ -1125,8 +1163,8 @@ return format_str; } - static void register_dtype(const std::initializer_list& fields) { - register_structured_dtype(fields, typeid(typename std::remove_cv::type), + static void register_dtype(any_container fields) { + register_structured_dtype(std::move(fields), typeid(typename std::remove_cv::type), sizeof(T), &direct_converter); } @@ -1199,7 +1237,8 @@ #define PYBIND11_NUMPY_DTYPE(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + (::std::vector<::pybind11::detail::field_descriptor> \ + {PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) #ifdef _MSC_VER #define PYBIND11_MAP2_LIST_NEXT1(test, next) \ @@ -1220,7 +1259,8 @@ #define PYBIND11_NUMPY_DTYPE_EX(Type, ...) \ ::pybind11::detail::npy_format_descriptor::register_dtype \ - ({PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) + (::std::vector<::pybind11::detail::field_descriptor> \ + {PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) #endif // __CLION_IDE__ @@ -1458,7 +1498,10 @@ private: remove_reference_t f; - template using param_n_t = typename pack_element::call_type...>::type; + // Internal compiler error in MSVC 19.16.27025.1 (Visual Studio 2017 15.9.4), when compiling with "/permissive-" flag + // when arg_call_types is manually inlined. + using arg_call_types = std::tuple::call_type...>; + template using param_n_t = typename std::tuple_element::type; // Runs a vectorized function given arguments tuple and three index sequences: // - Index is the full set of 0 ... (N-1) argument indices; @@ -1498,7 +1541,7 @@ if (trivial == broadcast_trivial::f_trivial) result = array_t(shape); else result = array_t(shape); - if (size == 0) return result; + if (size == 0) return std::move(result); /* Call the function */ if (trivial == broadcast_trivial::non_trivial) @@ -1506,7 +1549,7 @@ else apply_trivial(buffers, params, result.mutable_data(), size, i_seq, vi_seq, bi_seq); - return result; + return std::move(result); } template @@ -1559,9 +1602,7 @@ } template struct handle_type_name> { - static PYBIND11_DESCR name() { - return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); - } + static constexpr auto name = _("numpy.ndarray[") + npy_format_descriptor::name + _("]"); }; NAMESPACE_END(detail) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/pybind11.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/pybind11.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/pybind11.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/pybind11.h 2019-10-11 09:23:26.000000000 +0000 @@ -10,7 +10,17 @@ #pragma once -#if defined(_MSC_VER) +#if defined(__INTEL_COMPILER) +# pragma warning push +# pragma warning disable 68 // integer conversion resulted in a change of sign +# pragma warning disable 186 // pointless comparison of unsigned integer with zero +# pragma warning disable 878 // incompatible exception specifications +# pragma warning disable 1334 // the "template" keyword used for syntactic disambiguation may only be used within a template +# pragma warning disable 1682 // implicit conversion of a 64-bit integral type to a smaller integral type (potential portability problem) +# pragma warning disable 1786 // function "strdup" was declared deprecated +# pragma warning disable 1875 // offsetof applied to non-POD (Plain Old Data) types is nonstandard +# pragma warning disable 2196 // warning #2196: routine is both "inline" and "noinline" +#elif defined(_MSC_VER) # pragma warning(push) # pragma warning(disable: 4100) // warning C4100: Unreferenced formal parameter # pragma warning(disable: 4127) // warning C4127: Conditional expression is constant @@ -19,15 +29,6 @@ # pragma warning(disable: 4996) // warning C4996: The POSIX name for this item is deprecated. Instead, use the ISO C and C++ conformant name # pragma warning(disable: 4702) // warning C4702: unreachable code # pragma warning(disable: 4522) // warning C4522: multiple assignment operators specified -#elif defined(__INTEL_COMPILER) -# pragma warning(push) -# pragma warning(disable: 68) // integer conversion resulted in a change of sign -# pragma warning(disable: 186) // pointless comparison of unsigned integer with zero -# pragma warning(disable: 878) // incompatible exception specifications -# pragma warning(disable: 1334) // the "template" keyword used for syntactic disambiguation may only be used within a template -# pragma warning(disable: 1682) // implicit conversion of a 64-bit integral type to a smaller integral type (potential portability problem) -# pragma warning(disable: 1875) // offsetof applied to non-POD (Plain Old Data) types is nonstandard -# pragma warning(disable: 2196) // warning #2196: routine is both "inline" and "noinline" #elif defined(__GNUG__) && !defined(__clang__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-but-set-parameter" @@ -45,12 +46,17 @@ #include "detail/class.h" #include "detail/init.h" +#if defined(__GNUG__) && !defined(__clang__) +# include +#endif + NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object class cpp_function : public function { public: cpp_function() { } + cpp_function(std::nullptr_t) { } /// Construct a cpp_function from a vanilla function pointer template @@ -93,7 +99,6 @@ template void initialize(Func &&f, Return (*)(Args...), const Extra&... extra) { using namespace detail; - struct capture { remove_reference_t f; }; /* Store the function including any extra state it might have (e.g. a lambda capture object) */ @@ -164,10 +169,11 @@ process_attributes::init(extra..., rec); /* Generate a readable signature describing the function's arguments and return value types */ - PYBIND11_DESCR signature = _("(") + cast_in::arg_names() + _(") -> ") + cast_out::name(); + static constexpr auto signature = _("(") + cast_in::arg_names + _(") -> ") + cast_out::name; + PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ - initialize_generic(rec, signature.text(), signature.types(), sizeof...(Args)); + initialize_generic(rec, signature.text, types.data(), sizeof...(Args)); if (cast_in::has_args) rec->has_args = true; if (cast_in::has_kwargs) rec->has_kwargs = true; @@ -217,34 +223,30 @@ /* Generate a proper function signature */ std::string signature; - size_t type_depth = 0, char_index = 0, type_index = 0, arg_index = 0; - while (true) { - char c = text[char_index++]; - if (c == '\0') - break; + size_t type_index = 0, arg_index = 0; + for (auto *pc = text; *pc != '\0'; ++pc) { + const auto c = *pc; if (c == '{') { - // Write arg name for everything except *args, **kwargs and return type. - if (type_depth == 0 && text[char_index] != '*' && arg_index < args) { - if (!rec->args.empty() && rec->args[arg_index].name) { - signature += rec->args[arg_index].name; - } else if (arg_index == 0 && rec->is_method) { - signature += "self"; - } else { - signature += "arg" + std::to_string(arg_index - (rec->is_method ? 1 : 0)); - } - signature += ": "; + // Write arg name for everything except *args and **kwargs. + if (*(pc + 1) == '*') + continue; + + if (arg_index < rec->args.size() && rec->args[arg_index].name) { + signature += rec->args[arg_index].name; + } else if (arg_index == 0 && rec->is_method) { + signature += "self"; + } else { + signature += "arg" + std::to_string(arg_index - (rec->is_method ? 1 : 0)); } - ++type_depth; + signature += ": "; } else if (c == '}') { - --type_depth; - if (type_depth == 0) { - if (arg_index < rec->args.size() && rec->args[arg_index].descr) { - signature += "="; - signature += rec->args[arg_index].descr; - } - arg_index++; + // Write default value if available. + if (arg_index < rec->args.size() && rec->args[arg_index].descr) { + signature += " = "; + signature += rec->args[arg_index].descr; } + arg_index++; } else if (c == '%') { const std::type_info *t = types[type_index++]; if (!t) @@ -269,14 +271,9 @@ signature += c; } } - if (type_depth != 0 || types[type_index] != nullptr) + if (arg_index != args || types[type_index] != nullptr) pybind11_fail("Internal error while parsing type signature (2)"); - #if !defined(PYBIND11_CONSTEXPR_DESCR) - delete[] types; - delete[] text; - #endif - #if PY_MAJOR_VERSION < 3 if (strcmp(rec->name, "__next__") == 0) { std::free(rec->name); @@ -428,8 +425,8 @@ using namespace detail; /* Iterator over the list of potentially admissible overloads */ - function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), - *it = overloads; + const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + *it = overloads; /* Need to know how many arguments + keyword arguments there are to pick the right overload */ const size_t n_args_in = (size_t) PyTuple_GET_SIZE(args_in); @@ -485,7 +482,7 @@ result other than PYBIND11_TRY_NEXT_OVERLOAD. */ - function_record &func = *it; + const function_record &func = *it; size_t pos_args = func.nargs; // Number of positional arguments that we need if (func.has_args) --pos_args; // (but don't count py::args if (func.has_kwargs) --pos_args; // or py::kwargs) @@ -498,7 +495,7 @@ function_call call(func, parent); - size_t args_to_copy = std::min(pos_args, n_args_in); + size_t args_to_copy = (std::min)(pos_args, n_args_in); // Protect std::min with parentheses size_t args_copied = 0; // 0. Inject new-style `self` argument @@ -517,7 +514,7 @@ // 1. Copy any position arguments given. bool bad_arg = false; for (; args_copied < args_to_copy; ++args_copied) { - argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; + const argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; if (kwargs_in && arg_rec && arg_rec->name && PyDict_GetItemString(kwargs_in, arg_rec->name)) { bad_arg = true; break; @@ -658,13 +655,22 @@ result = PYBIND11_TRY_NEXT_OVERLOAD; } - if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) + if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { + // The error reporting logic below expects 'it' to be valid, as it would be + // if we'd encountered this failure in the first-pass loop. + if (!result) + it = &call.func; break; + } } } } catch (error_already_set &e) { e.restore(); return nullptr; +#if defined(__GNUG__) && !defined(__clang__) + } catch ( abi::__forced_unwind& ) { + throw; +#endif } catch (...) { /* When an exception is caught, give each registered exception translator a chance to translate it to a Python exception @@ -711,7 +717,7 @@ " arguments. The following argument types are supported:\n"; int ctr = 0; - for (function_record *it2 = overloads; it2 != nullptr; it2 = it2->next) { + for (const function_record *it2 = overloads; it2 != nullptr; it2 = it2->next) { msg += " "+ std::to_string(++ctr) + ". "; bool wrote_sig = false; @@ -899,6 +905,7 @@ tinfo->type = (PyTypeObject *) m_ptr; tinfo->cpptype = rec.type; tinfo->type_size = rec.type_size; + tinfo->type_align = rec.type_align; tinfo->operator_new = rec.operator_new; tinfo->holder_size_in_ptrs = size_in_ptrs(rec.holder_size); tinfo->init_instance = rec.init_instance; @@ -961,18 +968,18 @@ tinfo->get_buffer_data = get_buffer_data; } + // rec_func must be set for either fget or fset. void def_property_static_impl(const char *name, handle fget, handle fset, - detail::function_record *rec_fget) { - const auto is_static = !(rec_fget->is_method && rec_fget->scope); - const auto has_doc = rec_fget->doc && pybind11::options::show_user_defined_docstrings(); - + detail::function_record *rec_func) { + const auto is_static = rec_func && !(rec_func->is_method && rec_func->scope); + const auto has_doc = rec_func && rec_func->doc && pybind11::options::show_user_defined_docstrings(); auto property = handle((PyObject *) (is_static ? get_internals().static_property_type : &PyProperty_Type)); attr(name) = property(fget.ptr() ? fget : none(), fset.ptr() ? fset : none(), /*deleter*/none(), - pybind11::str(has_doc ? rec_fget->doc : "")); + pybind11::str(has_doc ? rec_func->doc : "")); } }; @@ -990,11 +997,21 @@ : std::true_type { }; /// Call class-specific delete if it exists or global otherwise. Can also be an overload set. template ::value, int> = 0> -void call_operator_delete(T *p, size_t) { T::operator delete(p); } +void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } template ::value && has_operator_delete_size::value, int> = 0> -void call_operator_delete(T *p, size_t s) { T::operator delete(p, s); } +void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } -inline void call_operator_delete(void *p, size_t) { ::operator delete(p); } +inline void call_operator_delete(void *p, size_t s, size_t a) { + (void)s; (void)a; +#if defined(PYBIND11_CPP17) + if (a > __STDCPP_DEFAULT_NEW_ALIGNMENT__) + ::operator delete(p, s, std::align_val_t(a)); + else + ::operator delete(p, s); +#else + ::operator delete(p); +#endif +} NAMESPACE_END(detail) @@ -1004,10 +1021,18 @@ auto method_adaptor(F &&f) -> decltype(std::forward(f)) { return std::forward(f); } template -auto method_adaptor(Return (Class::*pmf)(Args...)) -> Return (Derived::*)(Args...) { return pmf; } +auto method_adaptor(Return (Class::*pmf)(Args...)) -> Return (Derived::*)(Args...) { + static_assert(detail::is_accessible_base_of::value, + "Cannot bind an inaccessible base class method; use a lambda definition instead"); + return pmf; +} template -auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)(Args...) const { return pmf; } +auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)(Args...) const { + static_assert(detail::is_accessible_base_of::value, + "Cannot bind an inaccessible base class method; use a lambda definition instead"); + return pmf; +} template class class_ : public detail::generic_type { @@ -1049,10 +1074,11 @@ record.name = name; record.type = &typeid(type); record.type_size = sizeof(conditional_t); + record.type_align = alignof(conditional_t&); record.holder_size = sizeof(holder_type); record.init_instance = init_instance; record.dealloc = dealloc; - record.default_holder = std::is_same>::value; + record.default_holder = detail::is_instantiation::value; set_operator_new(&record); @@ -1094,7 +1120,7 @@ "def_static(...) called with a non-static member function pointer"); cpp_function cf(std::forward(f), name(name_), scope(*this), sibling(getattr(*this, name_, none())), extra...); - attr(cf.name()) = cf; + attr(cf.name()) = staticmethod(cf); return *this; } @@ -1158,7 +1184,7 @@ template class_ &def_readwrite(const char *name, D C::*pm, const Extra&... extra) { - static_assert(std::is_base_of::value, "def_readwrite() requires a class member (or base class member)"); + static_assert(std::is_same::value || std::is_base_of::value, "def_readwrite() requires a class member (or base class member)"); cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this)), fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); def_property(name, fget, fset, return_value_policy::reference_internal, extra...); @@ -1167,7 +1193,7 @@ template class_ &def_readonly(const char *name, const D C::*pm, const Extra& ...extra) { - static_assert(std::is_base_of::value, "def_readonly() requires a class member (or base class member)"); + static_assert(std::is_same::value || std::is_base_of::value, "def_readonly() requires a class member (or base class member)"); cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this)); def_property_readonly(name, fget, return_value_policy::reference_internal, extra...); return *this; @@ -1198,7 +1224,7 @@ /// Uses cpp_function's return_value_policy by default template class_ &def_property_readonly(const char *name, const cpp_function &fget, const Extra& ...extra) { - return def_property(name, fget, cpp_function(), extra...); + return def_property(name, fget, nullptr, extra...); } /// Uses return_value_policy::reference by default @@ -1210,7 +1236,7 @@ /// Uses cpp_function's return_value_policy by default template class_ &def_property_readonly_static(const char *name, const cpp_function &fget, const Extra& ...extra) { - return def_property_static(name, fget, cpp_function(), extra...); + return def_property_static(name, fget, nullptr, extra...); } /// Uses return_value_policy::reference_internal by default @@ -1239,22 +1265,28 @@ /// Uses cpp_function's return_value_policy by default template class_ &def_property_static(const char *name, const cpp_function &fget, const cpp_function &fset, const Extra& ...extra) { + static_assert( 0 == detail::constexpr_sum(std::is_base_of::value...), + "Argument annotations are not allowed for properties"); auto rec_fget = get_function_record(fget), rec_fset = get_function_record(fset); - char *doc_prev = rec_fget->doc; /* 'extra' field may include a property-specific documentation string */ - detail::process_attributes::init(extra..., rec_fget); - if (rec_fget->doc && rec_fget->doc != doc_prev) { - free(doc_prev); - rec_fget->doc = strdup(rec_fget->doc); + auto *rec_active = rec_fget; + if (rec_fget) { + char *doc_prev = rec_fget->doc; /* 'extra' field may include a property-specific documentation string */ + detail::process_attributes::init(extra..., rec_fget); + if (rec_fget->doc && rec_fget->doc != doc_prev) { + free(doc_prev); + rec_fget->doc = strdup(rec_fget->doc); + } } if (rec_fset) { - doc_prev = rec_fset->doc; + char *doc_prev = rec_fset->doc; detail::process_attributes::init(extra..., rec_fset); if (rec_fset->doc && rec_fset->doc != doc_prev) { free(doc_prev); rec_fset->doc = strdup(rec_fset->doc); } + if (! rec_active) rec_active = rec_fset; } - def_property_static_impl(name, fget, fset, rec_fget); + def_property_static_impl(name, fget, fset, rec_active); return *this; } @@ -1320,7 +1352,10 @@ v_h.set_holder_constructed(false); } else { - detail::call_operator_delete(v_h.value_ptr(), v_h.type->type_size); + detail::call_operator_delete(v_h.value_ptr(), + v_h.type->type_size, + v_h.type->type_align + ); } v_h.value_ptr() = nullptr; } @@ -1356,93 +1391,205 @@ return {std::forward(g), std::forward(s)}; } +NAMESPACE_BEGIN(detail) +struct enum_base { + enum_base(handle base, handle parent) : m_base(base), m_parent(parent) { } + + PYBIND11_NOINLINE void init(bool is_arithmetic, bool is_convertible) { + m_base.attr("__entries") = dict(); + auto property = handle((PyObject *) &PyProperty_Type); + auto static_property = handle((PyObject *) get_internals().static_property_type); + + m_base.attr("__repr__") = cpp_function( + [](handle arg) -> str { + handle type = arg.get_type(); + object type_name = type.attr("__name__"); + dict entries = type.attr("__entries"); + for (const auto &kv : entries) { + object other = kv.second[int_(0)]; + if (other.equal(arg)) + return pybind11::str("{}.{}").format(type_name, kv.first); + } + return pybind11::str("{}.???").format(type_name); + }, is_method(m_base) + ); + + m_base.attr("name") = property(cpp_function( + [](handle arg) -> str { + dict entries = arg.get_type().attr("__entries"); + for (const auto &kv : entries) { + if (handle(kv.second[int_(0)]).equal(arg)) + return pybind11::str(kv.first); + } + return "???"; + }, is_method(m_base) + )); + + m_base.attr("__doc__") = static_property(cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) + docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; + docstring += "Members:"; + for (const auto &kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n " + key; + if (!comment.is_none()) + docstring += " : " + (std::string) pybind11::str(comment); + } + return docstring; + } + ), none(), none(), ""); + + m_base.attr("__members__") = static_property(cpp_function( + [](handle arg) -> dict { + dict entries = arg.attr("__entries"), m; + for (const auto &kv : entries) + m[kv.first] = kv.second[int_(0)]; + return m; + }), none(), none(), "" + ); + + #define PYBIND11_ENUM_OP_STRICT(op, expr, strict_behavior) \ + m_base.attr(op) = cpp_function( \ + [](object a, object b) { \ + if (!a.get_type().is(b.get_type())) \ + strict_behavior; \ + return expr; \ + }, \ + is_method(m_base)) + + #define PYBIND11_ENUM_OP_CONV(op, expr) \ + m_base.attr(op) = cpp_function( \ + [](object a_, object b_) { \ + int_ a(a_), b(b_); \ + return expr; \ + }, \ + is_method(m_base)) + + #define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ + m_base.attr(op) = cpp_function( \ + [](object a_, object b) { \ + int_ a(a_); \ + return expr; \ + }, \ + is_method(m_base)) + + if (is_convertible) { + PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); + PYBIND11_ENUM_OP_CONV_LHS("__ne__", b.is_none() || !a.equal(b)); + + if (is_arithmetic) { + PYBIND11_ENUM_OP_CONV("__lt__", a < b); + PYBIND11_ENUM_OP_CONV("__gt__", a > b); + PYBIND11_ENUM_OP_CONV("__le__", a <= b); + PYBIND11_ENUM_OP_CONV("__ge__", a >= b); + PYBIND11_ENUM_OP_CONV("__and__", a & b); + PYBIND11_ENUM_OP_CONV("__rand__", a & b); + PYBIND11_ENUM_OP_CONV("__or__", a | b); + PYBIND11_ENUM_OP_CONV("__ror__", a | b); + PYBIND11_ENUM_OP_CONV("__xor__", a ^ b); + PYBIND11_ENUM_OP_CONV("__rxor__", a ^ b); + m_base.attr("__invert__") = cpp_function( + [](object arg) { return ~(int_(arg)); }, is_method(m_base)); + } + } else { + PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false); + PYBIND11_ENUM_OP_STRICT("__ne__", !int_(a).equal(int_(b)), return true); + + if (is_arithmetic) { + #define PYBIND11_THROW throw type_error("Expected an enumeration of matching type!"); + PYBIND11_ENUM_OP_STRICT("__lt__", int_(a) < int_(b), PYBIND11_THROW); + PYBIND11_ENUM_OP_STRICT("__gt__", int_(a) > int_(b), PYBIND11_THROW); + PYBIND11_ENUM_OP_STRICT("__le__", int_(a) <= int_(b), PYBIND11_THROW); + PYBIND11_ENUM_OP_STRICT("__ge__", int_(a) >= int_(b), PYBIND11_THROW); + #undef PYBIND11_THROW + } + } + + #undef PYBIND11_ENUM_OP_CONV_LHS + #undef PYBIND11_ENUM_OP_CONV + #undef PYBIND11_ENUM_OP_STRICT + + object getstate = cpp_function( + [](object arg) { return int_(arg); }, is_method(m_base)); + + m_base.attr("__getstate__") = getstate; + m_base.attr("__hash__") = getstate; + } + + PYBIND11_NOINLINE void value(char const* name_, object value, const char *doc = nullptr) { + dict entries = m_base.attr("__entries"); + str name(name_); + if (entries.contains(name)) { + std::string type_name = (std::string) str(m_base.attr("__name__")); + throw value_error(type_name + ": element \"" + std::string(name_) + "\" already exists!"); + } + + entries[name] = std::make_pair(value, doc); + m_base.attr(name) = value; + } + + PYBIND11_NOINLINE void export_values() { + dict entries = m_base.attr("__entries"); + for (const auto &kv : entries) + m_parent.attr(kv.first) = kv.second[int_(0)]; + } + + handle m_base; + handle m_parent; +}; + +NAMESPACE_END(detail) + /// Binds C++ enumerations and enumeration classes to Python template class enum_ : public class_ { public: - using class_::def; - using class_::def_property_readonly_static; + using Base = class_; + using Base::def; + using Base::attr; + using Base::def_property_readonly; + using Base::def_property_readonly_static; using Scalar = typename std::underlying_type::type; template enum_(const handle &scope, const char *name, const Extra&... extra) - : class_(scope, name, extra...), m_entries(), m_parent(scope) { - + : class_(scope, name, extra...), m_base(*this, scope) { constexpr bool is_arithmetic = detail::any_of...>::value; + constexpr bool is_convertible = std::is_convertible::value; + m_base.init(is_arithmetic, is_convertible); - auto m_entries_ptr = m_entries.inc_ref().ptr(); - def("__repr__", [name, m_entries_ptr](Type value) -> pybind11::str { - for (const auto &kv : reinterpret_borrow(m_entries_ptr)) { - if (pybind11::cast(kv.second) == value) - return pybind11::str("{}.{}").format(name, kv.first); - } - return pybind11::str("{}.???").format(name); - }); - def_property_readonly_static("__members__", [m_entries_ptr](object /* self */) { - dict m; - for (const auto &kv : reinterpret_borrow(m_entries_ptr)) - m[kv.first] = kv.second; - return m; - }, return_value_policy::copy); def(init([](Scalar i) { return static_cast(i); })); def("__int__", [](Type value) { return (Scalar) value; }); #if PY_MAJOR_VERSION < 3 def("__long__", [](Type value) { return (Scalar) value; }); #endif - def("__eq__", [](const Type &value, Type *value2) { return value2 && value == *value2; }); - def("__ne__", [](const Type &value, Type *value2) { return !value2 || value != *value2; }); - if (is_arithmetic) { - def("__lt__", [](const Type &value, Type *value2) { return value2 && value < *value2; }); - def("__gt__", [](const Type &value, Type *value2) { return value2 && value > *value2; }); - def("__le__", [](const Type &value, Type *value2) { return value2 && value <= *value2; }); - def("__ge__", [](const Type &value, Type *value2) { return value2 && value >= *value2; }); - } - if (std::is_convertible::value) { - // Don't provide comparison with the underlying type if the enum isn't convertible, - // i.e. if Type is a scoped enum, mirroring the C++ behaviour. (NB: we explicitly - // convert Type to Scalar below anyway because this needs to compile). - def("__eq__", [](const Type &value, Scalar value2) { return (Scalar) value == value2; }); - def("__ne__", [](const Type &value, Scalar value2) { return (Scalar) value != value2; }); - if (is_arithmetic) { - def("__lt__", [](const Type &value, Scalar value2) { return (Scalar) value < value2; }); - def("__gt__", [](const Type &value, Scalar value2) { return (Scalar) value > value2; }); - def("__le__", [](const Type &value, Scalar value2) { return (Scalar) value <= value2; }); - def("__ge__", [](const Type &value, Scalar value2) { return (Scalar) value >= value2; }); - def("__invert__", [](const Type &value) { return ~((Scalar) value); }); - def("__and__", [](const Type &value, Scalar value2) { return (Scalar) value & value2; }); - def("__or__", [](const Type &value, Scalar value2) { return (Scalar) value | value2; }); - def("__xor__", [](const Type &value, Scalar value2) { return (Scalar) value ^ value2; }); - def("__rand__", [](const Type &value, Scalar value2) { return (Scalar) value & value2; }); - def("__ror__", [](const Type &value, Scalar value2) { return (Scalar) value | value2; }); - def("__rxor__", [](const Type &value, Scalar value2) { return (Scalar) value ^ value2; }); - def("__and__", [](const Type &value, const Type &value2) { return (Scalar) value & (Scalar) value2; }); - def("__or__", [](const Type &value, const Type &value2) { return (Scalar) value | (Scalar) value2; }); - def("__xor__", [](const Type &value, const Type &value2) { return (Scalar) value ^ (Scalar) value2; }); - } - } - def("__hash__", [](const Type &value) { return (Scalar) value; }); - // Pickling and unpickling -- needed for use with the 'multiprocessing' module - def(pickle([](const Type &value) { return pybind11::make_tuple((Scalar) value); }, - [](tuple t) { return static_cast(t[0].cast()); })); + #if PY_MAJOR_VERSION > 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) + def("__index__", [](Type value) { return (Scalar) value; }); + #endif + + cpp_function setstate( + [](Type &value, Scalar arg) { value = static_cast(arg); }, + is_method(*this)); + attr("__setstate__") = setstate; } /// Export enumeration entries into the parent scope enum_& export_values() { - for (const auto &kv : m_entries) - m_parent.attr(kv.first) = kv.second; + m_base.export_values(); return *this; } /// Add an enumeration entry - enum_& value(char const* name, Type value) { - auto v = pybind11::cast(value, return_value_policy::copy); - this->attr(name) = v; - m_entries[pybind11::str(name)] = v; + enum_& value(char const* name, Type value, const char *doc = nullptr) { + m_base.value(name, pybind11::cast(value, return_value_policy::copy), doc); return *this; } private: - dict m_entries; - handle m_parent; + detail::enum_base m_base; }; NAMESPACE_BEGIN(detail) @@ -1750,6 +1897,15 @@ tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); if (!tstate) { + /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if + calling from a Python thread). Since we use a different key, this ensures + we don't create a new thread state and deadlock in PyEval_AcquireThread + below. Note we don't save this state with internals.tstate, since we don't + create it we would fail to clear it (its reference count should be > 0). */ + tstate = PyGILState_GetThisThreadState(); + } + + if (!tstate) { tstate = PyThreadState_New(internals.istate); #if !defined(NDEBUG) if (!tstate) @@ -1856,12 +2012,12 @@ #endif error_already_set::~error_already_set() { - if (type) { - error_scope scope; + if (m_type) { gil_scoped_acquire gil; - type.release().dec_ref(); - value.release().dec_ref(); - trace.release().dec_ref(); + error_scope scope; + m_type.release().dec_ref(); + m_value.release().dec_ref(); + m_trace.release().dec_ref(); } } @@ -1922,6 +2078,14 @@ return overload; } +/** \rst + Try to retrieve a python method by the provided name from the instance pointed to by the this_ptr. + + :this_ptr: The pointer to the object the overload should be retrieved for. This should be the first + non-trampoline class encountered in the inheritance chain. + :name: The name of the overloaded Python method to retrieve. + :return: The Python method by this name from the object or an empty function wrapper. + \endrst */ template function get_overload(const T *this_ptr, const char *name) { auto tinfo = detail::get_type_info(typeid(T)); return tinfo ? get_type_overload(this_ptr, tinfo, name) : function(); @@ -1940,26 +2104,73 @@ } \ } +/** \rst + Macro to populate the virtual method in the trampoline class. This macro tries to look up a method named 'fn' + from the Python side, deals with the :ref:`gil` and necessary argument conversions to call this method and return + the appropriate type. See :ref:`overriding_virtuals` for more information. This macro should be used when the method + name in C is not the same as the method name in Python. For example with `__str__`. + + .. code-block:: cpp + + std::string toString() override { + PYBIND11_OVERLOAD_NAME( + std::string, // Return type (ret_type) + Animal, // Parent class (cname) + toString, // Name of function in C++ (name) + "__str__", // Name of method in Python (fn) + ); + } +\endrst */ #define PYBIND11_OVERLOAD_NAME(ret_type, cname, name, fn, ...) \ - PYBIND11_OVERLOAD_INT(ret_type, cname, name, __VA_ARGS__) \ + PYBIND11_OVERLOAD_INT(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) \ return cname::fn(__VA_ARGS__) +/** \rst + Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERLOAD_NAME`, except that it + throws if no overload can be found. +\endrst */ #define PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, name, fn, ...) \ - PYBIND11_OVERLOAD_INT(ret_type, cname, name, __VA_ARGS__) \ - pybind11::pybind11_fail("Tried to call pure virtual function \"" #cname "::" name "\""); + PYBIND11_OVERLOAD_INT(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) \ + pybind11::pybind11_fail("Tried to call pure virtual function \"" PYBIND11_STRINGIFY(cname) "::" name "\""); +/** \rst + Macro to populate the virtual method in the trampoline class. This macro tries to look up the method + from the Python side, deals with the :ref:`gil` and necessary argument conversions to call this method and return + the appropriate type. This macro should be used if the method name in C and in Python are identical. + See :ref:`overriding_virtuals` for more information. + + .. code-block:: cpp + + class PyAnimal : public Animal { + public: + // Inherit the constructors + using Animal::Animal; + + // Trampoline (need one for each virtual function) + std::string go(int n_times) override { + PYBIND11_OVERLOAD_PURE( + std::string, // Return type (ret_type) + Animal, // Parent class (cname) + go, // Name of function in C++ (must match Python name) (fn) + n_times // Argument(s) (...) + ); + } + }; +\endrst */ #define PYBIND11_OVERLOAD(ret_type, cname, fn, ...) \ - PYBIND11_OVERLOAD_NAME(ret_type, cname, #fn, fn, __VA_ARGS__) + PYBIND11_OVERLOAD_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) +/** \rst + Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERLOAD`, except that it throws + if no overload can be found. +\endrst */ #define PYBIND11_OVERLOAD_PURE(ret_type, cname, fn, ...) \ - PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, #fn, fn, __VA_ARGS__) + PYBIND11_OVERLOAD_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) NAMESPACE_END(PYBIND11_NAMESPACE) -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) # pragma warning(pop) -#elif defined(__INTEL_COMPILER) -/* Leave ignored warnings on */ #elif defined(__GNUG__) && !defined(__clang__) # pragma GCC diagnostic pop #endif diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/pytypes.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/pytypes.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/pytypes.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/pytypes.h 2019-10-08 08:35:40.000000000 +0000 @@ -114,6 +114,35 @@ bool is(object_api const& other) const { return derived().ptr() == other.derived().ptr(); } /// Equivalent to ``obj is None`` in Python. bool is_none() const { return derived().ptr() == Py_None; } + /// Equivalent to obj == other in Python + bool equal(object_api const &other) const { return rich_compare(other, Py_EQ); } + bool not_equal(object_api const &other) const { return rich_compare(other, Py_NE); } + bool operator<(object_api const &other) const { return rich_compare(other, Py_LT); } + bool operator<=(object_api const &other) const { return rich_compare(other, Py_LE); } + bool operator>(object_api const &other) const { return rich_compare(other, Py_GT); } + bool operator>=(object_api const &other) const { return rich_compare(other, Py_GE); } + + object operator-() const; + object operator~() const; + object operator+(object_api const &other) const; + object operator+=(object_api const &other) const; + object operator-(object_api const &other) const; + object operator-=(object_api const &other) const; + object operator*(object_api const &other) const; + object operator*=(object_api const &other) const; + object operator/(object_api const &other) const; + object operator/=(object_api const &other) const; + object operator|(object_api const &other) const; + object operator|=(object_api const &other) const; + object operator&(object_api const &other) const; + object operator&=(object_api const &other) const; + object operator^(object_api const &other) const; + object operator^=(object_api const &other) const; + object operator<<(object_api const &other) const; + object operator<<=(object_api const &other) const; + object operator>>(object_api const &other) const; + object operator>>=(object_api const &other) const; + PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; @@ -124,6 +153,9 @@ int ref_count() const { return static_cast(Py_REFCNT(derived().ptr())); } /// Return a handle to the Python type object underlying the instance handle get_type() const; + +private: + bool rich_compare(object_api const &other, int value) const; }; NAMESPACE_END(detail) @@ -292,15 +324,18 @@ /// Constructs a new exception from the current Python error indicator, if any. The current /// Python error indicator will be cleared. error_already_set() : std::runtime_error(detail::error_string()) { - PyErr_Fetch(&type.ptr(), &value.ptr(), &trace.ptr()); + PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); } + error_already_set(const error_already_set &) = default; + error_already_set(error_already_set &&) = default; + inline ~error_already_set(); /// Give the currently-held error back to Python, if any. If there is currently a Python error /// already set it is cleared first. After this call, the current object no longer stores the /// error variables (but the `.what()` string is still available). - void restore() { PyErr_Restore(type.release().ptr(), value.release().ptr(), trace.release().ptr()); } + void restore() { PyErr_Restore(m_type.release().ptr(), m_value.release().ptr(), m_trace.release().ptr()); } // Does nothing; provided for backwards compatibility. PYBIND11_DEPRECATED("Use of error_already_set.clear() is deprecated") @@ -309,10 +344,14 @@ /// Check if the currently trapped error type matches the given Python exception class (or a /// subclass thereof). May also be passed a tuple to search for any exception class matches in /// the given tuple. - bool matches(handle ex) const { return PyErr_GivenExceptionMatches(ex.ptr(), type.ptr()); } + bool matches(handle exc) const { return PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()); } + + const object& type() const { return m_type; } + const object& value() const { return m_value; } + const object& trace() const { return m_trace; } private: - object type, value, trace; + object m_type, m_value, m_trace; }; /** \defgroup python_builtins _ @@ -353,6 +392,14 @@ return PyObject_HasAttrString(obj.ptr(), name) == 1; } +inline void delattr(handle obj, handle name) { + if (PyObject_DelAttr(obj.ptr(), name.ptr()) != 0) { throw error_already_set(); } +} + +inline void delattr(handle obj, const char *name) { + if (PyObject_DelAttrString(obj.ptr(), name) != 0) { throw error_already_set(); } +} + inline object getattr(handle obj, handle name) { PyObject *result = PyObject_GetAttr(obj.ptr(), name.ptr()); if (!result) { throw error_already_set(); } @@ -424,7 +471,6 @@ // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } - template class accessor : public object_api> { using key_type = typename Policy::key_type; @@ -662,7 +708,7 @@ private: handle obj; - PyObject *key, *value; + PyObject *key = nullptr, *value = nullptr; ssize_t pos = -1; }; NAMESPACE_END(iterator_policies) @@ -690,9 +736,14 @@ } inline bool PyNone_Check(PyObject *o) { return o == Py_None; } +#if PY_MAJOR_VERSION >= 3 +inline bool PyEllipsis_Check(PyObject *o) { return o == Py_Ellipsis; } +#endif inline bool PyUnicode_Check_Permissive(PyObject *o) { return PyUnicode_Check(o) || PYBIND11_BYTES_CHECK(o); } +inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } + class kwargs_proxy : public handle { public: explicit kwargs_proxy(handle h) : handle(h) { } @@ -964,6 +1015,14 @@ none() : object(Py_None, borrowed_t{}) { } }; +#if PY_MAJOR_VERSION >= 3 +class ellipsis : public object { +public: + PYBIND11_OBJECT(ellipsis, object, detail::PyEllipsis_Check) + ellipsis() : object(Py_Ellipsis, borrowed_t{}) { } +}; +#endif + class bool_ : public object { public: PYBIND11_OBJECT_CVT(bool_, object, PyBool_Check, raw_bool) @@ -1074,6 +1133,13 @@ (ssize_t *) stop, (ssize_t *) step, (ssize_t *) slicelength) == 0; } + bool compute(ssize_t length, ssize_t *start, ssize_t *stop, ssize_t *step, + ssize_t *slicelength) const { + return PySlice_GetIndicesEx((PYBIND11_SLICE_OBJECT *) m_ptr, + length, start, + stop, step, + slicelength) == 0; + } }; class capsule : public object { @@ -1136,7 +1202,9 @@ if (!m_ptr) pybind11_fail("Could not allocate tuple object!"); } size_t size() const { return (size_t) PyTuple_Size(m_ptr); } + bool empty() const { return size() == 0; } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } + detail::item_accessor operator[](handle h) const { return object::operator[](h); } detail::tuple_iterator begin() const { return {*this, 0}; } detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; @@ -1154,11 +1222,13 @@ explicit dict(Args &&...args) : dict(collector(std::forward(args)...).kwargs()) { } size_t size() const { return (size_t) PyDict_Size(m_ptr); } + bool empty() const { return size() == 0; } detail::dict_iterator begin() const { return {*this, 0}; } detail::dict_iterator end() const { return {}; } void clear() const { PyDict_Clear(ptr()); } - bool contains(handle key) const { return PyDict_Contains(ptr(), key.ptr()) == 1; } - bool contains(const char *key) const { return PyDict_Contains(ptr(), pybind11::str(key).ptr()) == 1; } + template bool contains(T &&key) const { + return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward(key)).ptr()) == 1; + } private: /// Call the `dict` Python type -- always returns a new reference @@ -1173,7 +1243,9 @@ public: PYBIND11_OBJECT_DEFAULT(sequence, object, PySequence_Check) size_t size() const { return (size_t) PySequence_Size(m_ptr); } + bool empty() const { return size() == 0; } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } + detail::item_accessor operator[](handle h) const { return object::operator[](h); } detail::sequence_iterator begin() const { return {*this, 0}; } detail::sequence_iterator end() const { return {*this, PySequence_Size(m_ptr)}; } }; @@ -1185,12 +1257,18 @@ if (!m_ptr) pybind11_fail("Could not allocate list object!"); } size_t size() const { return (size_t) PyList_Size(m_ptr); } + bool empty() const { return size() == 0; } detail::list_accessor operator[](size_t index) const { return {*this, index}; } + detail::item_accessor operator[](handle h) const { return object::operator[](h); } detail::list_iterator begin() const { return {*this, 0}; } detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } template void append(T &&val) const { PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()); } + template void insert(size_t index, T &&val) const { + PyList_Insert(m_ptr, static_cast(index), + detail::object_or_cast(std::forward(val)).ptr()); + } }; class args : public tuple { PYBIND11_OBJECT_DEFAULT(args, tuple, PyTuple_Check) }; @@ -1203,10 +1281,14 @@ if (!m_ptr) pybind11_fail("Could not allocate set object!"); } size_t size() const { return (size_t) PySet_Size(m_ptr); } + bool empty() const { return size() == 0; } template bool add(T &&val) const { return PySet_Add(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) == 0; } void clear() const { PySet_Clear(m_ptr); } + template bool contains(T &&val) const { + return PySet_Contains(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) == 1; + } }; class function : public object { @@ -1221,11 +1303,16 @@ bool is_cpp_function() const { return (bool) cpp_function(); } }; +class staticmethod : public object { +public: + PYBIND11_OBJECT_CVT(staticmethod, object, detail::PyStaticMethod_Check, PyStaticMethod_New) +}; + class buffer : public object { public: PYBIND11_OBJECT_DEFAULT(buffer, object, PyObject_CheckBuffer) - buffer_info request(bool writable = false) { + buffer_info request(bool writable = false) const { int flags = PyBUF_STRIDES | PyBUF_FORMAT; if (writable) flags |= PyBUF_WRITABLE; Py_buffer *view = new Py_buffer(); @@ -1279,6 +1366,21 @@ return (size_t) result; } +inline size_t len_hint(handle h) { +#if PY_VERSION_HEX >= 0x03040000 + ssize_t result = PyObject_LengthHint(h.ptr(), 0); +#else + ssize_t result = PyObject_Length(h.ptr()); +#endif + if (result < 0) { + // Sometimes a length can't be determined at all (eg generators) + // In which case simply return 0 + PyErr_Clear(); + return 0; + } + return (size_t) result; +} + inline str repr(handle h) { PyObject *str_value = PyObject_Repr(h.ptr()); if (!str_value) throw error_already_set(); @@ -1328,5 +1430,55 @@ template handle object_api::get_type() const { return (PyObject *) Py_TYPE(derived().ptr()); } +template +bool object_api::rich_compare(object_api const &other, int value) const { + int rv = PyObject_RichCompareBool(derived().ptr(), other.derived().ptr(), value); + if (rv == -1) + throw error_already_set(); + return rv == 1; +} + +#define PYBIND11_MATH_OPERATOR_UNARY(op, fn) \ + template object object_api::op() const { \ + object result = reinterpret_steal(fn(derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + +#define PYBIND11_MATH_OPERATOR_BINARY(op, fn) \ + template \ + object object_api::op(object_api const &other) const { \ + object result = reinterpret_steal( \ + fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + +PYBIND11_MATH_OPERATOR_UNARY (operator~, PyNumber_Invert) +PYBIND11_MATH_OPERATOR_UNARY (operator-, PyNumber_Negative) +PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) +PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd) +PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) +PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract) +PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) +PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply) +PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) +PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide) +PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) +PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr) +PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) +PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd) +PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) +PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor) +PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) +PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift) +PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) +PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift) + +#undef PYBIND11_MATH_OPERATOR_UNARY +#undef PYBIND11_MATH_OPERATOR_BINARY + NAMESPACE_END(detail) NAMESPACE_END(PYBIND11_NAMESPACE) diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/stl_bind.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/stl_bind.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/stl_bind.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/stl_bind.h 2019-10-08 08:35:40.000000000 +0000 @@ -115,6 +115,14 @@ using SizeType = typename Vector::size_type; using DiffType = typename Vector::difference_type; + auto wrap_i = [](DiffType i, SizeType n) { + if (i < 0) + i += n; + if (i < 0 || (SizeType)i >= n) + throw index_error(); + return i; + }; + cl.def("append", [](Vector &v, const T &value) { v.push_back(value); }, arg("x"), @@ -122,7 +130,7 @@ cl.def(init([](iterable it) { auto v = std::unique_ptr(new Vector()); - v->reserve(len(it)); + v->reserve(len_hint(it)); for (handle h : it) v->push_back(h.cast()); return v.release(); @@ -136,11 +144,36 @@ "Extend the list by appending all the items in the given list" ); + cl.def("extend", + [](Vector &v, iterable it) { + const size_t old_size = v.size(); + v.reserve(old_size + len_hint(it)); + try { + for (handle h : it) { + v.push_back(h.cast()); + } + } catch (const cast_error &) { + v.erase(v.begin() + static_cast(old_size), v.end()); + try { + v.shrink_to_fit(); + } catch (const std::exception &) { + // Do nothing + } + throw; + } + }, + arg("L"), + "Extend the list by appending all the items in the given list" + ); + cl.def("insert", - [](Vector &v, SizeType i, const T &x) { - if (i > v.size()) + [](Vector &v, DiffType i, const T &x) { + // Can't use wrap_i; i == v.size() is OK + if (i < 0) + i += v.size(); + if (i < 0 || (SizeType)i > v.size()) throw index_error(); - v.insert(v.begin() + (DiffType) i, x); + v.insert(v.begin() + i, x); }, arg("i") , arg("x"), "Insert an item at a given position." @@ -158,11 +191,10 @@ ); cl.def("pop", - [](Vector &v, SizeType i) { - if (i >= v.size()) - throw index_error(); - T t = v[i]; - v.erase(v.begin() + (DiffType) i); + [wrap_i](Vector &v, DiffType i) { + i = wrap_i(i, v.size()); + T t = v[(SizeType) i]; + v.erase(v.begin() + i); return t; }, arg("i"), @@ -170,10 +202,9 @@ ); cl.def("__setitem__", - [](Vector &v, SizeType i, const T &t) { - if (i >= v.size()) - throw index_error(); - v[i] = t; + [wrap_i](Vector &v, DiffType i, const T &t) { + i = wrap_i(i, v.size()); + v[(SizeType)i] = t; } ); @@ -216,10 +247,9 @@ ); cl.def("__delitem__", - [](Vector &v, SizeType i) { - if (i >= v.size()) - throw index_error(); - v.erase(v.begin() + DiffType(i)); + [wrap_i](Vector &v, DiffType i) { + i = wrap_i(i, v.size()); + v.erase(v.begin() + i); }, "Delete the list elements at index ``i``" ); @@ -255,13 +285,21 @@ void vector_accessor(enable_if_t::value, Class_> &cl) { using T = typename Vector::value_type; using SizeType = typename Vector::size_type; + using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; + auto wrap_i = [](DiffType i, SizeType n) { + if (i < 0) + i += n; + if (i < 0 || (SizeType)i >= n) + throw index_error(); + return i; + }; + cl.def("__getitem__", - [](Vector &v, SizeType i) -> T & { - if (i >= v.size()) - throw index_error(); - return v[i]; + [wrap_i](Vector &v, DiffType i) -> T & { + i = wrap_i(i, v.size()); + return v[(SizeType)i]; }, return_value_policy::reference_internal // ref + keepalive ); @@ -281,12 +319,15 @@ void vector_accessor(enable_if_t::value, Class_> &cl) { using T = typename Vector::value_type; using SizeType = typename Vector::size_type; + using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; cl.def("__getitem__", - [](const Vector &v, SizeType i) -> T { - if (i >= v.size()) + [](const Vector &v, DiffType i) -> T { + if (i < 0 && (i += v.size()) < 0) throw index_error(); - return v[i]; + if ((SizeType)i >= v.size()) + throw index_error(); + return v[(SizeType)i]; } ); @@ -579,6 +620,15 @@ return_value_policy::reference_internal // ref + keepalive ); + cl.def("__contains__", + [](Map &m, const KeyType &k) -> bool { + auto it = m.find(k); + if (it == m.end()) + return false; + return true; + } + ); + // Assignment provided only if the type is copyable detail::map_assignment(cl); diff -Nru spead2-1.10.0/3rdparty/pybind11/include/pybind11/stl.h spead2-2.1.0/3rdparty/pybind11/include/pybind11/stl.h --- spead2-1.10.0/3rdparty/pybind11/include/pybind11/stl.h 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/include/pybind11/stl.h 2019-10-08 08:35:40.000000000 +0000 @@ -16,6 +16,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) @@ -83,7 +84,8 @@ template static handle cast(T &&src, return_value_policy policy, handle parent) { - policy = return_value_policy_override::policy(policy); + if (!std::is_lvalue_reference::value) + policy = return_value_policy_override::policy(policy); pybind11::set s; for (auto &&value : src) { auto value_ = reinterpret_steal(key_conv::cast(forward_like(value), policy, parent)); @@ -93,7 +95,7 @@ return s.release(); } - PYBIND11_TYPE_CASTER(type, _("Set[") + key_conv::name() + _("]")); + PYBIND11_TYPE_CASTER(type, _("Set[") + key_conv::name + _("]")); }; template struct map_caster { @@ -119,8 +121,12 @@ template static handle cast(T &&src, return_value_policy policy, handle parent) { dict d; - return_value_policy policy_key = return_value_policy_override::policy(policy); - return_value_policy policy_value = return_value_policy_override::policy(policy); + return_value_policy policy_key = policy; + return_value_policy policy_value = policy; + if (!std::is_lvalue_reference::value) { + policy_key = return_value_policy_override::policy(policy_key); + policy_value = return_value_policy_override::policy(policy_value); + } for (auto &&kv : src) { auto key = reinterpret_steal(key_conv::cast(forward_like(kv.first), policy_key, parent)); auto value = reinterpret_steal(value_conv::cast(forward_like(kv.second), policy_value, parent)); @@ -131,14 +137,14 @@ return d.release(); } - PYBIND11_TYPE_CASTER(Type, _("Dict[") + key_conv::name() + _(", ") + value_conv::name() + _("]")); + PYBIND11_TYPE_CASTER(Type, _("Dict[") + key_conv::name + _(", ") + value_conv::name + _("]")); }; template struct list_caster { using value_conv = make_caster; bool load(handle src, bool convert) { - if (!isinstance(src)) + if (!isinstance(src) || isinstance(src)) return false; auto s = reinterpret_borrow(src); value.clear(); @@ -161,7 +167,8 @@ public: template static handle cast(T &&src, return_value_policy policy, handle parent) { - policy = return_value_policy_override::policy(policy); + if (!std::is_lvalue_reference::value) + policy = return_value_policy_override::policy(policy); list l(src.size()); size_t index = 0; for (auto &&value : src) { @@ -173,12 +180,15 @@ return l.release(); } - PYBIND11_TYPE_CASTER(Type, _("List[") + value_conv::name() + _("]")); + PYBIND11_TYPE_CASTER(Type, _("List[") + value_conv::name + _("]")); }; template struct type_caster> : list_caster, Type> { }; +template struct type_caster> + : list_caster, Type> { }; + template struct type_caster> : list_caster, Type> { }; @@ -199,9 +209,9 @@ public: bool load(handle src, bool convert) { - if (!isinstance(src)) + if (!isinstance(src)) return false; - auto l = reinterpret_borrow(src); + auto l = reinterpret_borrow(src); if (!require_size(l.size())) return false; size_t ctr = 0; @@ -227,7 +237,7 @@ return l.release(); } - PYBIND11_TYPE_CASTER(ArrayType, _("List[") + value_conv::name() + _(_(""), _("[") + _() + _("]")) + _("]")); + PYBIND11_TYPE_CASTER(ArrayType, _("List[") + value_conv::name + _(_(""), _("[") + _() + _("]")) + _("]")); }; template struct type_caster> @@ -274,7 +284,7 @@ return true; } - PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name() + _("]")); + PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]")); }; #if PYBIND11_HAS_OPTIONAL @@ -354,7 +364,7 @@ } using Type = V; - PYBIND11_TYPE_CASTER(Type, _("Union[") + detail::concat(make_caster::name()...) + _("]")); + PYBIND11_TYPE_CASTER(Type, _("Union[") + detail::concat(make_caster::name...) + _("]")); }; #if PYBIND11_HAS_VARIANT diff -Nru spead2-1.10.0/3rdparty/pybind11/pybind11/__init__.py spead2-2.1.0/3rdparty/pybind11/pybind11/__init__.py --- spead2-1.10.0/3rdparty/pybind11/pybind11/__init__.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/pybind11/__init__.py 2019-10-08 08:35:40.000000000 +0000 @@ -10,9 +10,17 @@ virtualenv = hasattr(sys, 'real_prefix') or \ sys.prefix != getattr(sys, "base_prefix", sys.prefix) + # Are we running in a conda environment? + conda = os.path.exists(os.path.join(sys.prefix, 'conda-meta')) + if virtualenv: return os.path.join(sys.prefix, 'include', 'site', 'python' + sys.version[:3]) + elif conda: + if os.name == 'nt': + return os.path.join(sys.prefix, 'Library', 'include') + else: + return os.path.join(sys.prefix, 'include') else: dist = Distribution({'name': 'pybind11'}) dist.parse_config_files() diff -Nru spead2-1.10.0/3rdparty/pybind11/pybind11/_version.py spead2-2.1.0/3rdparty/pybind11/pybind11/_version.py --- spead2-1.10.0/3rdparty/pybind11/pybind11/_version.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/pybind11/_version.py 2019-10-11 09:23:26.000000000 +0000 @@ -1,2 +1,2 @@ -version_info = (2, 2, 4) +version_info = (2, 4, 2) __version__ = '.'.join(map(str, version_info)) diff -Nru spead2-1.10.0/3rdparty/pybind11/README.md spead2-2.1.0/3rdparty/pybind11/README.md --- spead2-1.10.0/3rdparty/pybind11/README.md 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/README.md 2019-10-08 08:35:40.000000000 +0000 @@ -51,7 +51,6 @@ - Custom operators - Single and multiple inheritance - STL data structures -- Iterators and ranges - Smart pointers with reference counting like ``std::shared_ptr`` - Internal references with correct reference counting - C++ classes with virtual (and pure virtual) methods can be extended in Python @@ -87,9 +86,8 @@ [reported](http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf) a binary size reduction of **5.4x** and compile time reduction by **5.8x**. -- When supported by the compiler, two new C++14 features (relaxed constexpr and - return value deduction) are used to precompute function signatures at compile - time, leading to smaller binaries. +- Function signatures are precomputed at compile time (using ``constexpr``), + leading to smaller binaries. - With little extra effort, C++ types can be pickled and unpickled similar to regular Python objects. @@ -107,6 +105,7 @@ This project was created by [Wenzel Jakob](http://rgl.epfl.ch/people/wjakob). Significant features and/or improvements to the code were contributed by Jonas Adler, +Lori A. Burns, Sylvain Corlay, Trent Houliston, Axel Huebl, @@ -119,6 +118,7 @@ Jason Rhinelander, Boris Schäling, Pim Schellart, +Henry Schreiner, Ivan Smirnov, and Patrick Stewart. diff -Nru spead2-1.10.0/3rdparty/pybind11/setup.cfg spead2-2.1.0/3rdparty/pybind11/setup.cfg --- spead2-1.10.0/3rdparty/pybind11/setup.cfg 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/setup.cfg 2019-10-08 08:35:40.000000000 +0000 @@ -7,4 +7,6 @@ exclude = .git, __pycache__, build, dist, docs, tools, venv ignore = # required for pretty matrix formatting: multiple spaces after `,` and `[` - E201, E241 + E201, E241, W504, + # camelcase 'cPickle' imported as lowercase 'pickle' + N813 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/CMakeLists.txt spead2-2.1.0/3rdparty/pybind11/tests/CMakeLists.txt --- spead2-1.10.0/3rdparty/pybind11/tests/CMakeLists.txt 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/CMakeLists.txt 2019-10-08 08:35:40.000000000 +0000 @@ -26,6 +26,7 @@ # Full set of test files (you can override these; see below) set(PYBIND11_TEST_FILES + test_async.cpp test_buffers.cpp test_builtin_casters.cpp test_call_policies.cpp @@ -40,6 +41,7 @@ test_eval.cpp test_exceptions.cpp test_factory_constructors.cpp + test_gil_scoped.cpp test_iostream.cpp test_kwargs_and_defaults.cpp test_local_bindings.cpp @@ -57,6 +59,8 @@ test_smart_ptr.cpp test_stl.cpp test_stl_binders.cpp + test_tagbased_polymorphic.cpp + test_union.cpp test_virtual_functions.cpp ) @@ -68,6 +72,13 @@ set(PYBIND11_TEST_FILES ${PYBIND11_TEST_OVERRIDE}) endif() +# Skip test_async for Python < 3.5 +list(FIND PYBIND11_TEST_FILES test_async.cpp PYBIND11_TEST_FILES_ASYNC_I) +if((PYBIND11_TEST_FILES_ASYNC_I GREATER -1) AND ("${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" VERSION_LESS 3.5)) + message(STATUS "Skipping test_async because Python version ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} < 3.5") + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_ASYNC_I}) +endif() + string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}") # Contains the set of test files that require pybind11_cross_module_tests to be @@ -80,6 +91,10 @@ test_stl_binders.py ) +set(PYBIND11_CROSS_MODULE_GIL_TESTS + test_gil_scoped.py +) + # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" # skip message). @@ -89,7 +104,7 @@ # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also # produces a fatal error if loaded from a pre-3.0 cmake. if (NOT CMAKE_VERSION VERSION_LESS 3.0) - find_package(Eigen3 QUIET CONFIG) + find_package(Eigen3 3.2.7 QUIET CONFIG) if (EIGEN3_FOUND) if (EIGEN3_VERSION_STRING AND NOT EIGEN3_VERSION_STRING VERSION_LESS 3.3.1) set(PYBIND11_EIGEN_VIA_TARGET 1) @@ -99,7 +114,7 @@ if (NOT EIGEN3_FOUND) # Couldn't load via target, so fall back to allowing module mode finding, which will pick up # tools/FindEigen3.cmake - find_package(Eigen3 QUIET) + find_package(Eigen3 3.2.7 QUIET) endif() if(EIGEN3_FOUND) @@ -123,14 +138,14 @@ function(pybind11_enable_warnings target_name) if(MSVC) target_compile_options(${target_name} PRIVATE /W4) - else() - target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)") + target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated) endif() if(PYBIND11_WERROR) if(MSVC) target_compile_options(${target_name} PRIVATE /WX) - else() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)") target_compile_options(${target_name} PRIVATE -Werror) endif() endif() @@ -146,6 +161,14 @@ break() endif() endforeach() + +foreach(t ${PYBIND11_CROSS_MODULE_GIL_TESTS}) + list(FIND PYBIND11_PYTEST_FILES ${t} i) + if (i GREATER -1) + list(APPEND test_targets cross_module_gil_utils) + break() + endif() +endforeach() set(testdir ${CMAKE_CURRENT_SOURCE_DIR}) foreach(target ${test_targets}) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/conftest.py spead2-2.1.0/3rdparty/pybind11/tests/conftest.py --- spead2-1.10.0/3rdparty/pybind11/tests/conftest.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/conftest.py 2019-10-08 08:35:40.000000000 +0000 @@ -17,6 +17,11 @@ _long_marker = re.compile(r'([0-9])L') _hexadecimal = re.compile(r'0x[0-9a-fA-F]+') +# test_async.py requires support for async and await +collect_ignore = [] +if sys.version_info[:2] < (3, 5): + collect_ignore.append("test_async.py") + def _strip_and_dedent(s): """For triple-quote strings""" @@ -75,7 +80,7 @@ self.capfd.readouterr() return self - def __exit__(self, *_): + def __exit__(self, *args): self.out, self.err = self.capfd.readouterr() def __eq__(self, other): @@ -185,7 +190,7 @@ gc.collect() -def pytest_namespace(): +def pytest_configure(): """Add import suppression and test requirements to `pytest` namespace""" try: import numpy as np @@ -202,19 +207,17 @@ pypy = platform.python_implementation() == "PyPy" skipif = pytest.mark.skipif - return { - 'suppress': suppress, - 'requires_numpy': skipif(not np, reason="numpy is not installed"), - 'requires_scipy': skipif(not np, reason="scipy is not installed"), - 'requires_eigen_and_numpy': skipif(not have_eigen or not np, - reason="eigen and/or numpy are not installed"), - 'requires_eigen_and_scipy': skipif(not have_eigen or not scipy, - reason="eigen and/or scipy are not installed"), - 'unsupported_on_pypy': skipif(pypy, reason="unsupported on PyPy"), - 'unsupported_on_py2': skipif(sys.version_info.major < 3, - reason="unsupported on Python 2.x"), - 'gc_collect': gc_collect - } + pytest.suppress = suppress + pytest.requires_numpy = skipif(not np, reason="numpy is not installed") + pytest.requires_scipy = skipif(not np, reason="scipy is not installed") + pytest.requires_eigen_and_numpy = skipif(not have_eigen or not np, + reason="eigen and/or numpy are not installed") + pytest.requires_eigen_and_scipy = skipif( + not have_eigen or not scipy, reason="eigen and/or scipy are not installed") + pytest.unsupported_on_pypy = skipif(pypy, reason="unsupported on PyPy") + pytest.unsupported_on_py2 = skipif(sys.version_info.major < 3, + reason="unsupported on Python 2.x") + pytest.gc_collect = gc_collect def _test_import_pybind11(): diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/constructor_stats.h spead2-2.1.0/3rdparty/pybind11/tests/constructor_stats.h --- spead2-1.10.0/3rdparty/pybind11/tests/constructor_stats.h 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/constructor_stats.h 2019-10-08 08:35:40.000000000 +0000 @@ -180,7 +180,7 @@ } } } - catch (std::out_of_range) {} + catch (const std::out_of_range &) {} if (!t1) throw std::runtime_error("Unknown class passed to ConstructorStats::get()"); auto &cs1 = get(*t1); // If we have both a t1 and t2 match, one is probably the trampoline class; return whichever diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/cross_module_gil_utils.cpp spead2-2.1.0/3rdparty/pybind11/tests/cross_module_gil_utils.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/cross_module_gil_utils.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/cross_module_gil_utils.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,73 @@ +/* + tests/cross_module_gil_utils.cpp -- tools for acquiring GIL from a different module + + Copyright (c) 2019 Google LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ +#include +#include + +// This file mimics a DSO that makes pybind11 calls but does not define a +// PYBIND11_MODULE. The purpose is to test that such a DSO can create a +// py::gil_scoped_acquire when the running thread is in a GIL-released state. +// +// Note that we define a Python module here for convenience, but in general +// this need not be the case. The typical scenario would be a DSO that implements +// shared logic used internally by multiple pybind11 modules. + +namespace { + +namespace py = pybind11; +void gil_acquire() { py::gil_scoped_acquire gil; } + +constexpr char kModuleName[] = "cross_module_gil_utils"; + +#if PY_MAJOR_VERSION >= 3 +struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + kModuleName, + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + NULL +}; +#else +PyMethodDef module_methods[] = { + {NULL, NULL, 0, NULL} +}; +#endif + +} // namespace + +extern "C" PYBIND11_EXPORT +#if PY_MAJOR_VERSION >= 3 +PyObject* PyInit_cross_module_gil_utils() +#else +void initcross_module_gil_utils() +#endif +{ + + PyObject* m = +#if PY_MAJOR_VERSION >= 3 + PyModule_Create(&moduledef); +#else + Py_InitModule(kModuleName, module_methods); +#endif + + if (m != NULL) { + static_assert( + sizeof(&gil_acquire) == sizeof(void*), + "Function pointer must have the same size as void*"); + PyModule_AddObject(m, "gil_acquire_funcaddr", + PyLong_FromVoidPtr(reinterpret_cast(&gil_acquire))); + } + +#if PY_MAJOR_VERSION >= 3 + return m; +#endif +} diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/pytest.ini spead2-2.1.0/3rdparty/pybind11/tests/pytest.ini --- spead2-1.10.0/3rdparty/pybind11/tests/pytest.ini 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/pytest.ini 2019-10-08 08:35:40.000000000 +0000 @@ -13,3 +13,4 @@ ignore::ImportWarning # bogus numpy ABI warning (see numpy/#432) ignore:.*numpy.dtype size changed.*:RuntimeWarning + ignore:.*numpy.ufunc size changed.*:RuntimeWarning diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_async.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_async.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_async.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_async.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,26 @@ +/* + tests/test_async.cpp -- __await__ support + + Copyright (c) 2019 Google Inc. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "pybind11_tests.h" + +TEST_SUBMODULE(async_module, m) { + struct DoesNotSupportAsync {}; + py::class_(m, "DoesNotSupportAsync") + .def(py::init<>()); + struct SupportsAsync {}; + py::class_(m, "SupportsAsync") + .def(py::init<>()) + .def("__await__", [](const SupportsAsync& self) -> py::object { + static_cast(self); + py::object loop = py::module::import("asyncio.events").attr("get_event_loop")(); + py::object f = loop.attr("create_future")(); + f.attr("set_result")(5); + return f.attr("__await__")(); + }); +} diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_async.py spead2-2.1.0/3rdparty/pybind11/tests/test_async.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_async.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_async.py 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,23 @@ +import asyncio +import pytest +from pybind11_tests import async_module as m + + +@pytest.fixture +def event_loop(): + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +async def get_await_result(x): + return await x + + +def test_await(event_loop): + assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync())) + + +def test_await_missing(event_loop): + with pytest.raises(TypeError): + event_loop.run_until_complete(get_await_result(m.DoesNotSupportAsync())) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_buffers.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_buffers.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_buffers.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_buffers.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -78,7 +78,7 @@ py::class_(m, "Matrix", py::buffer_protocol()) .def(py::init()) /// Construct from a buffer - .def(py::init([](py::buffer b) { + .def(py::init([](py::buffer const b) { py::buffer_info info = b.request(); if (info.format != py::format_descriptor::format() || info.ndim != 2) throw std::runtime_error("Incompatible buffer format!"); @@ -107,7 +107,7 @@ return py::buffer_info( m.data(), /* Pointer to buffer */ { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * size_t(m.rows()), /* Strides (in bytes) for each index */ + { sizeof(float) * size_t(m.cols()), /* Strides (in bytes) for each index */ sizeof(float) } ); }) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_buffers.py spead2-2.1.0/3rdparty/pybind11/tests/test_buffers.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_buffers.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_buffers.py 2019-10-08 08:35:40.000000000 +0000 @@ -36,17 +36,21 @@ # https://bitbucket.org/pypy/pypy/issues/2444 @pytest.unsupported_on_pypy def test_to_python(): - mat = m.Matrix(5, 5) - assert memoryview(mat).shape == (5, 5) + mat = m.Matrix(5, 4) + assert memoryview(mat).shape == (5, 4) assert mat[2, 3] == 0 - mat[2, 3] = 4 + mat[2, 3] = 4.0 + mat[3, 2] = 7.0 assert mat[2, 3] == 4 + assert mat[3, 2] == 7 + assert struct.unpack_from('f', mat, (3 * 4 + 2) * 4) == (7, ) + assert struct.unpack_from('f', mat, (2 * 4 + 3) * 4) == (4, ) mat2 = np.array(mat, copy=False) - assert mat2.shape == (5, 5) - assert abs(mat2).sum() == 4 - assert mat2[2, 3] == 4 + assert mat2.shape == (5, 4) + assert abs(mat2).sum() == 11 + assert mat2[2, 3] == 4 and mat2[3, 2] == 7 mat2[2, 3] = 5 assert mat2[2, 3] == 5 @@ -58,7 +62,7 @@ del mat2 # holds a mat reference pytest.gc_collect() assert cstats.alive() == 0 - assert cstats.values() == ["5x5 matrix"] + assert cstats.values() == ["5x4 matrix"] assert cstats.copy_constructions == 0 # assert cstats.move_constructions >= 0 # Don't invoke any assert cstats.copy_assignments == 0 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_builtin_casters.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_builtin_casters.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_builtin_casters.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_builtin_casters.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -155,4 +155,16 @@ // test_complex m.def("complex_cast", [](float x) { return "{}"_s.format(x); }); m.def("complex_cast", [](std::complex x) { return "({}, {})"_s.format(x.real(), x.imag()); }); + + // test int vs. long (Python 2) + m.def("int_cast", []() {return (int) 42;}); + m.def("long_cast", []() {return (long) 42;}); + m.def("longlong_cast", []() {return ULLONG_MAX;}); + + /// test void* cast operator + m.def("test_void_caster", []() -> bool { + void *v = (void *) 0xabcd; + py::object o = py::cast(v); + return py::cast(o) == v; + }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_builtin_casters.py spead2-2.1.0/3rdparty/pybind11/tests/test_builtin_casters.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_builtin_casters.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_builtin_casters.py 2019-10-08 08:35:40.000000000 +0000 @@ -323,3 +323,20 @@ assert convert(np.bool_(False)) is False assert noconvert(np.bool_(True)) is True assert noconvert(np.bool_(False)) is False + + +def test_int_long(): + """In Python 2, a C++ int should return a Python int rather than long + if possible: longs are not always accepted where ints are used (such + as the argument to sys.exit()). A C++ long long is always a Python + long.""" + + import sys + must_be_long = type(getattr(sys, 'maxint', 1) + 1) + assert isinstance(m.int_cast(), int) + assert isinstance(m.long_cast(), int) + assert isinstance(m.longlong_cast(), must_be_long) + + +def test_void_caster_2(): + assert m.test_void_caster() diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_callbacks.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_callbacks.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_callbacks.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_callbacks.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -10,6 +10,7 @@ #include "pybind11_tests.h" #include "constructor_stats.h" #include +#include int dummy_function(int i) { return i + 1; } @@ -146,4 +147,22 @@ py::class_(m, "CppBoundMethodTest") .def(py::init<>()) .def("triple", [](CppBoundMethodTest &, int val) { return 3 * val; }); + + // test async Python callbacks + using callback_f = std::function; + m.def("test_async_callback", [](callback_f f, py::list work) { + // make detached thread that calls `f` with piece of work after a little delay + auto start_f = [f](int j) { + auto invoke_f = [f, j] { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + f(j); + }; + auto t = std::thread(std::move(invoke_f)); + t.detach(); + }; + + // spawn worker threads + for (auto i : work) + start_f(py::cast(i)); + }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_callbacks.py spead2-2.1.0/3rdparty/pybind11/tests/test_callbacks.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_callbacks.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_callbacks.py 2019-10-08 08:35:40.000000000 +0000 @@ -1,5 +1,6 @@ import pytest from pybind11_tests import callbacks as m +from threading import Thread def test_callbacks(): @@ -105,3 +106,31 @@ def test_movable_object(): assert m.callback_with_movable(lambda _: None) is True + + +def test_async_callbacks(): + # serves as state for async callback + class Item: + def __init__(self, value): + self.value = value + + res = [] + + # generate stateful lambda that will store result in `res` + def gen_f(): + s = Item(3) + return lambda j: res.append(s.value + j) + + # do some work async + work = [1, 2, 3, 4] + m.test_async_callback(gen_f(), work) + # wait until work is done + from time import sleep + sleep(0.5) + assert sum(res) == sum([x + 3 for x in work]) + + +def test_async_async_callbacks(): + t = Thread(target=test_async_callbacks) + t.start() + t.join() diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_call_policies.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_call_policies.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_call_policies.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_call_policies.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -8,7 +8,6 @@ */ #include "pybind11_tests.h" -#include "constructor_stats.h" struct CustomGuard { static bool enabled; @@ -37,6 +36,8 @@ class Child { public: Child() { py::print("Allocating child."); } + Child(const Child &) = default; + Child(Child &&) = default; ~Child() { py::print("Releasing child."); } }; py::class_(m, "Child") @@ -60,21 +61,6 @@ .def("returnNullChildKeepAliveChild", &Parent::returnNullChild, py::keep_alive<1, 0>()) .def("returnNullChildKeepAliveParent", &Parent::returnNullChild, py::keep_alive<0, 1>()); - // test_keep_alive_single - m.def("add_patient", [](py::object /*nurse*/, py::object /*patient*/) { }, py::keep_alive<1, 2>()); - m.def("get_patients", [](py::object nurse) { - py::list patients; - for (PyObject *p : pybind11::detail::get_internals().patients[nurse.ptr()]) - patients.append(py::reinterpret_borrow(p)); - return patients; - }); - m.def("refcount", [](py::handle h) { -#ifdef PYPY_VERSION - ConstructorStats::gc(); // PyPy doesn't update ref counts until GC occurs -#endif - return h.ref_count(); - }); - #if !defined(PYPY_VERSION) // test_alive_gc class ParentGC : public Parent { diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_call_policies.py spead2-2.1.0/3rdparty/pybind11/tests/test_call_policies.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_call_policies.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_call_policies.py 2019-10-08 08:35:40.000000000 +0000 @@ -1,6 +1,6 @@ import pytest from pybind11_tests import call_policies as m -from pybind11_tests import ConstructorStats, UserType +from pybind11_tests import ConstructorStats def test_keep_alive_argument(capture): @@ -69,35 +69,6 @@ """ -def test_keep_alive_single(): - """Issue #1251 - patients are stored multiple times when given to the same nurse""" - - nurse, p1, p2 = UserType(), UserType(), UserType() - b = m.refcount(nurse) - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b, b] - m.add_patient(nurse, p1) - assert m.get_patients(nurse) == [p1, ] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b] - m.add_patient(nurse, p1) - assert m.get_patients(nurse) == [p1, ] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b] - m.add_patient(nurse, p1) - assert m.get_patients(nurse) == [p1, ] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b] - m.add_patient(nurse, p2) - assert m.get_patients(nurse) == [p1, p2] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b + 1] - m.add_patient(nurse, p2) - assert m.get_patients(nurse) == [p1, p2] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b + 1] - m.add_patient(nurse, p2) - m.add_patient(nurse, p1) - assert m.get_patients(nurse) == [p1, p2] - assert [m.refcount(nurse), m.refcount(p1), m.refcount(p2)] == [b, b + 1, b + 1] - del nurse - assert [m.refcount(p1), m.refcount(p2)] == [b, b] - - # https://bitbucket.org/pypy/pypy/issues/2447 @pytest.unsupported_on_pypy def test_alive_gc(capture): diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_chrono.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_chrono.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_chrono.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_chrono.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -14,6 +14,10 @@ TEST_SUBMODULE(chrono, m) { using system_time = std::chrono::system_clock::time_point; using steady_time = std::chrono::steady_clock::time_point; + + using timespan = std::chrono::duration; + using timestamp = std::chrono::time_point; + // test_chrono_system_clock // Return the current time off the wall clock m.def("test_chrono1", []() { return std::chrono::system_clock::now(); }); @@ -44,4 +48,8 @@ // Float durations (issue #719) m.def("test_chrono_float_diff", [](std::chrono::duration a, std::chrono::duration b) { return a - b; }); + + m.def("test_nano_timepoint", [](timestamp start, timespan delta) -> timestamp { + return start + delta; + }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_chrono.py spead2-2.1.0/3rdparty/pybind11/tests/test_chrono.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_chrono.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_chrono.py 2019-10-08 08:35:40.000000000 +0000 @@ -40,6 +40,62 @@ assert diff.microseconds == 0 +def test_chrono_system_clock_roundtrip_date(): + date1 = datetime.date.today() + + # Roundtrip the time + datetime2 = m.test_chrono2(date1) + date2 = datetime2.date() + time2 = datetime2.time() + + # The returned value should be a datetime + assert isinstance(datetime2, datetime.datetime) + assert isinstance(date2, datetime.date) + assert isinstance(time2, datetime.time) + + # They should be identical (no information lost on roundtrip) + diff = abs(date1 - date2) + assert diff.days == 0 + assert diff.seconds == 0 + assert diff.microseconds == 0 + + # Year, Month & Day should be the same after the round trip + assert date1.year == date2.year + assert date1.month == date2.month + assert date1.day == date2.day + + # There should be no time information + assert time2.hour == 0 + assert time2.minute == 0 + assert time2.second == 0 + assert time2.microsecond == 0 + + +def test_chrono_system_clock_roundtrip_time(): + time1 = datetime.datetime.today().time() + + # Roundtrip the time + datetime2 = m.test_chrono2(time1) + date2 = datetime2.date() + time2 = datetime2.time() + + # The returned value should be a datetime + assert isinstance(datetime2, datetime.datetime) + assert isinstance(date2, datetime.date) + assert isinstance(time2, datetime.time) + + # Hour, Minute, Second & Microsecond should be the same after the round trip + assert time1.hour == time2.hour + assert time1.minute == time2.minute + assert time1.second == time2.second + assert time1.microsecond == time2.microsecond + + # There should be no date information (i.e. date = python base date) + assert date2.year == 1970 + assert date2.month == 1 + assert date2.day == 1 + + def test_chrono_duration_roundtrip(): # Get the difference between two times (a timedelta) @@ -70,6 +126,19 @@ assert cpp_diff.microseconds == diff.microseconds +def test_chrono_duration_subtraction_equivalence_date(): + + date1 = datetime.date.today() + date2 = datetime.date.today() + + diff = date2 - date1 + cpp_diff = m.test_chrono4(date2, date1) + + assert cpp_diff.days == diff.days + assert cpp_diff.seconds == diff.seconds + assert cpp_diff.microseconds == diff.microseconds + + def test_chrono_steady_clock(): time1 = m.test_chrono5() assert isinstance(time1, datetime.timedelta) @@ -99,3 +168,9 @@ diff = m.test_chrono_float_diff(43.789012, 1.123456) assert diff.seconds == 42 assert 665556 <= diff.microseconds <= 665557 + + +def test_nano_timepoint(): + time = datetime.datetime.now() + time1 = m.test_nano_timepoint(time, datetime.timedelta(seconds=60)) + assert(time1 == time + datetime.timedelta(seconds=60)) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_class.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_class.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_class.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_class.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -12,6 +12,10 @@ #include "local_bindings.h" #include +#if defined(_MSC_VER) +# pragma warning(disable: 4324) // warning C4324: structure was padded due to alignment specifier +#endif + // test_brace_initialization struct NoBraceInitialization { NoBraceInitialization(std::vector v) : vec{std::move(v)} {} @@ -24,6 +28,9 @@ TEST_SUBMODULE(class_, m) { // test_instance struct NoConstructor { + NoConstructor() = default; + NoConstructor(const NoConstructor &) = default; + NoConstructor(NoConstructor &&) = default; static NoConstructor *new_instance() { auto *ptr = new NoConstructor(); print_created(ptr, "via new_instance"); @@ -92,7 +99,12 @@ m.def("dog_bark", [](const Dog &dog) { return dog.bark(); }); // test_automatic_upcasting - struct BaseClass { virtual ~BaseClass() {} }; + struct BaseClass { + BaseClass() = default; + BaseClass(const BaseClass &) = default; + BaseClass(BaseClass &&) = default; + virtual ~BaseClass() {} + }; struct DerivedClass1 : BaseClass { }; struct DerivedClass2 : BaseClass { }; @@ -333,6 +345,28 @@ "a"_a, "b"_a, "c"_a); base.def("g", [](NestBase &, Nested &) {}); base.def("h", []() { return NestBase(); }); + + // test_error_after_conversion + // The second-pass path through dispatcher() previously didn't + // remember which overload was used, and would crash trying to + // generate a useful error message + + struct NotRegistered {}; + struct StringWrapper { std::string str; }; + m.def("test_error_after_conversions", [](int) {}); + m.def("test_error_after_conversions", + [](StringWrapper) -> NotRegistered { return {}; }); + py::class_(m, "StringWrapper").def(py::init()); + py::implicitly_convertible(); + + #if defined(PYBIND11_CPP17) + struct alignas(1024) Aligned { + std::uintptr_t ptr() const { return (uintptr_t) this; } + }; + py::class_(m, "Aligned") + .def(py::init<>()) + .def("ptr", &Aligned::ptr); + #endif } template class BreaksBase { public: virtual ~BreaksBase() = default; }; diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_class.py spead2-2.1.0/3rdparty/pybind11/tests/test_class.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_class.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_class.py 2019-10-08 08:35:40.000000000 +0000 @@ -266,3 +266,16 @@ Invoked with: 0 ''' + + +def test_error_after_conversions(): + with pytest.raises(TypeError) as exc_info: + m.test_error_after_conversions("hello") + assert str(exc_info.value).startswith( + "Unable to convert function return value to a Python type!") + + +def test_aligned(): + if hasattr(m, "Aligned"): + p = m.Aligned().ptr() + assert p % 1024 == 0 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_constants_and_functions.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_constants_and_functions.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_constants_and_functions.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_constants_and_functions.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -49,7 +49,14 @@ int f1(int x) noexcept { return x+1; } int f2(int x) noexcept(true) { return x+2; } int f3(int x) noexcept(false) { return x+3; } +#if defined(__GNUG__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wdeprecated" +#endif int f4(int x) throw() { return x+4; } // Deprecated equivalent to noexcept(true) +#if defined(__GNUG__) +# pragma GCC diagnostic pop +#endif struct C { int m1(int x) noexcept { return x-1; } int m2(int x) const noexcept { return x-2; } @@ -57,8 +64,15 @@ int m4(int x) const noexcept(true) { return x-4; } int m5(int x) noexcept(false) { return x-5; } int m6(int x) const noexcept(false) { return x-6; } +#if defined(__GNUG__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wdeprecated" +#endif int m7(int x) throw() { return x-7; } int m8(int x) const throw() { return x-8; } +#if defined(__GNUG__) +# pragma GCC diagnostic pop +#endif }; } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_copy_move.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_copy_move.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_copy_move.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_copy_move.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -86,7 +86,7 @@ protected: CopyOnlyInt value; public: - static PYBIND11_DESCR name() { return _("CopyOnlyInt"); } + static constexpr auto name = _("CopyOnlyInt"); bool load(handle src, bool) { value = CopyOnlyInt(src.cast()); return true; } static handle cast(const CopyOnlyInt &m, return_value_policy r, handle p) { return pybind11::cast(m.value, r, p); } static handle cast(const CopyOnlyInt *src, return_value_policy policy, handle parent) { diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_eigen.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_eigen.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_eigen.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_eigen.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -124,7 +124,7 @@ // This one accepts a matrix of any stride: m.def("add_any", [](py::EigenDRef x, int r, int c, double v) { x(r,c) += v; }); - // Return mutable references (numpy maps into eigen varibles) + // Return mutable references (numpy maps into eigen variables) m.def("get_cm_ref", []() { return Eigen::Ref(get_cm()); }); m.def("get_rm_ref", []() { return Eigen::Ref(get_rm()); }); // The same references, but non-mutable (numpy maps into eigen variables, but is !writeable) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_eigen.py spead2-2.1.0/3rdparty/pybind11/tests/test_eigen.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_eigen.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_eigen.py 2019-10-08 08:35:40.000000000 +0000 @@ -679,10 +679,10 @@ # These should still fail (incompatible dimensions): with pytest.raises(TypeError) as excinfo: m.iss1105_row(np.ones((7, 1))) - assert "incompatible function arguments" in str(excinfo) + assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.iss1105_col(np.ones((1, 7))) - assert "incompatible function arguments" in str(excinfo) + assert "incompatible function arguments" in str(excinfo.value) def test_custom_operator_new(): diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_enum.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_enum.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_enum.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_enum.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -13,11 +13,13 @@ // test_unscoped_enum enum UnscopedEnum { EOne = 1, - ETwo + ETwo, + EThree }; - py::enum_(m, "UnscopedEnum", py::arithmetic()) - .value("EOne", EOne) - .value("ETwo", ETwo) + py::enum_(m, "UnscopedEnum", py::arithmetic(), "An unscoped enumeration") + .value("EOne", EOne, "Docstring for EOne") + .value("ETwo", ETwo, "Docstring for ETwo") + .value("EThree", EThree, "Docstring for EThree") .export_values(); // test_scoped_enum @@ -68,4 +70,18 @@ m.def("test_enum_to_int", [](int) { }); m.def("test_enum_to_uint", [](uint32_t) { }); m.def("test_enum_to_long_long", [](long long) { }); + + // test_duplicate_enum_name + enum SimpleEnum + { + ONE, TWO, THREE + }; + + m.def("register_bad_enum", [m]() { + py::enum_(m, "SimpleEnum") + .value("ONE", SimpleEnum::ONE) //NOTE: all value function calls are called with the same first parameter value + .value("ONE", SimpleEnum::TWO) + .value("ONE", SimpleEnum::THREE) + .export_values(); + }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_enum.py spead2-2.1.0/3rdparty/pybind11/tests/test_enum.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_enum.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_enum.py 2019-10-08 08:35:40.000000000 +0000 @@ -6,9 +6,22 @@ assert str(m.UnscopedEnum.EOne) == "UnscopedEnum.EOne" assert str(m.UnscopedEnum.ETwo) == "UnscopedEnum.ETwo" assert str(m.EOne) == "UnscopedEnum.EOne" + + # name property + assert m.UnscopedEnum.EOne.name == "EOne" + assert m.UnscopedEnum.ETwo.name == "ETwo" + assert m.EOne.name == "EOne" + # name readonly + with pytest.raises(AttributeError): + m.UnscopedEnum.EOne.name = "" + # name returns a copy + foo = m.UnscopedEnum.EOne.name + foo = "bar" + assert m.UnscopedEnum.EOne.name == "EOne" + # __members__ property assert m.UnscopedEnum.__members__ == \ - {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo} + {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo, "EThree": m.UnscopedEnum.EThree} # __members__ readonly with pytest.raises(AttributeError): m.UnscopedEnum.__members__ = {} @@ -16,12 +29,57 @@ foo = m.UnscopedEnum.__members__ foo["bar"] = "baz" assert m.UnscopedEnum.__members__ == \ - {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo} + {"EOne": m.UnscopedEnum.EOne, "ETwo": m.UnscopedEnum.ETwo, "EThree": m.UnscopedEnum.EThree} + + for docstring_line in '''An unscoped enumeration + +Members: + + EOne : Docstring for EOne + + ETwo : Docstring for ETwo - # no TypeError exception for unscoped enum ==/!= int comparisons + EThree : Docstring for EThree'''.split('\n'): + assert docstring_line in m.UnscopedEnum.__doc__ + + # Unscoped enums will accept ==/!= int comparisons y = m.UnscopedEnum.ETwo assert y == 2 + assert 2 == y assert y != 3 + assert 3 != y + # Compare with None + assert (y != None) # noqa: E711 + assert not (y == None) # noqa: E711 + # Compare with an object + assert (y != object()) + assert not (y == object()) + # Compare with string + assert y != "2" + assert "2" != y + assert not ("2" == y) + assert not (y == "2") + + with pytest.raises(TypeError): + y < object() + + with pytest.raises(TypeError): + y <= object() + + with pytest.raises(TypeError): + y > object() + + with pytest.raises(TypeError): + y >= object() + + with pytest.raises(TypeError): + y | object() + + with pytest.raises(TypeError): + y & object() + + with pytest.raises(TypeError): + y ^ object() assert int(m.UnscopedEnum.ETwo) == 2 assert str(m.UnscopedEnum(2)) == "UnscopedEnum.ETwo" @@ -40,17 +98,37 @@ assert not (m.UnscopedEnum.ETwo < m.UnscopedEnum.EOne) assert not (2 < m.UnscopedEnum.EOne) + # arithmetic + assert m.UnscopedEnum.EOne & m.UnscopedEnum.EThree == m.UnscopedEnum.EOne + assert m.UnscopedEnum.EOne | m.UnscopedEnum.ETwo == m.UnscopedEnum.EThree + assert m.UnscopedEnum.EOne ^ m.UnscopedEnum.EThree == m.UnscopedEnum.ETwo + def test_scoped_enum(): assert m.test_scoped_enum(m.ScopedEnum.Three) == "ScopedEnum::Three" z = m.ScopedEnum.Two assert m.test_scoped_enum(z) == "ScopedEnum::Two" - # expected TypeError exceptions for scoped enum ==/!= int comparisons + # Scoped enums will *NOT* accept ==/!= int comparisons (Will always return False) + assert not z == 3 + assert not 3 == z + assert z != 3 + assert 3 != z + # Compare with None + assert (z != None) # noqa: E711 + assert not (z == None) # noqa: E711 + # Compare with an object + assert (z != object()) + assert not (z == object()) + # Scoped enums will *NOT* accept >, <, >= and <= int comparisons (Will throw exceptions) + with pytest.raises(TypeError): + z > 3 with pytest.raises(TypeError): - assert z == 2 + z < 3 with pytest.raises(TypeError): - assert z != 3 + z >= 3 + with pytest.raises(TypeError): + z <= 3 # order assert m.ScopedEnum.Two < m.ScopedEnum.Three @@ -100,6 +178,7 @@ assert int(m.Flags.Read | m.Flags.Execute) == 5 assert int(m.Flags.Write | m.Flags.Execute) == 3 assert int(m.Flags.Write | 1) == 3 + assert ~m.Flags.Write == -3 state = m.Flags.Read | m.Flags.Write assert (state & m.Flags.Read) != 0 @@ -119,3 +198,9 @@ m.test_enum_to_uint(m.ClassWithUnscopedEnum.EMode.EFirstMode) m.test_enum_to_long_long(m.Flags.Read) m.test_enum_to_long_long(m.ClassWithUnscopedEnum.EMode.EFirstMode) + + +def test_duplicate_enum_name(): + with pytest.raises(ValueError) as excinfo: + m.register_bad_enum() + assert str(excinfo.value) == 'SimpleEnum: element "ONE" already exists!' diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_exceptions.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_exceptions.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_exceptions.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_exceptions.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -118,10 +118,38 @@ m.def("throws_logic_error", []() { throw std::logic_error("this error should fall through to the standard handler"); }); m.def("exception_matches", []() { py::dict foo; - try { foo["bar"]; } + try { + // Assign to a py::object to force read access of nonexistent dict entry + py::object o = foo["bar"]; + } catch (py::error_already_set& ex) { if (!ex.matches(PyExc_KeyError)) throw; + return true; + } + return false; + }); + m.def("exception_matches_base", []() { + py::dict foo; + try { + // Assign to a py::object to force read access of nonexistent dict entry + py::object o = foo["bar"]; + } + catch (py::error_already_set &ex) { + if (!ex.matches(PyExc_Exception)) throw; + return true; + } + return false; + }); + m.def("modulenotfound_exception_matches_base", []() { + try { + // On Python >= 3.6, this raises a ModuleNotFoundError, a subclass of ImportError + py::module::import("nonexistent"); + } + catch (py::error_already_set &ex) { + if (!ex.matches(PyExc_ImportError)) throw; + return true; } + return false; }); m.def("throw_already_set", [](bool err) { diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_exceptions.py spead2-2.1.0/3rdparty/pybind11/tests/test_exceptions.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_exceptions.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_exceptions.py 2019-10-08 08:35:40.000000000 +0000 @@ -48,7 +48,9 @@ def test_exception_matches(): - m.exception_matches() + assert m.exception_matches() + assert m.exception_matches_base() + assert m.modulenotfound_exception_matches_base() def test_custom(msg): diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_factory_constructors.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_factory_constructors.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_factory_constructors.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_factory_constructors.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -285,6 +285,7 @@ // test_reallocations // Class that has verbose operator_new/operator_delete calls struct NoisyAlloc { + NoisyAlloc(const NoisyAlloc &) = default; NoisyAlloc(int i) { py::print(py::str("NoisyAlloc(int {})").format(i)); } NoisyAlloc(double d) { py::print(py::str("NoisyAlloc(double {})").format(d)); } ~NoisyAlloc() { py::print("~NoisyAlloc()"); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_gil_scoped.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_gil_scoped.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_gil_scoped.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_gil_scoped.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,52 @@ +/* + tests/test_gil_scoped.cpp -- acquire and release gil + + Copyright (c) 2017 Borja Zarco (Google LLC) + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "pybind11_tests.h" +#include + + +class VirtClass { +public: + virtual ~VirtClass() {} + virtual void virtual_func() {} + virtual void pure_virtual_func() = 0; +}; + +class PyVirtClass : public VirtClass { + void virtual_func() override { + PYBIND11_OVERLOAD(void, VirtClass, virtual_func,); + } + void pure_virtual_func() override { + PYBIND11_OVERLOAD_PURE(void, VirtClass, pure_virtual_func,); + } +}; + +TEST_SUBMODULE(gil_scoped, m) { + py::class_(m, "VirtClass") + .def(py::init<>()) + .def("virtual_func", &VirtClass::virtual_func) + .def("pure_virtual_func", &VirtClass::pure_virtual_func); + + m.def("test_callback_py_obj", + [](py::object func) { func(); }); + m.def("test_callback_std_func", + [](const std::function &func) { func(); }); + m.def("test_callback_virtual_func", + [](VirtClass &virt) { virt.virtual_func(); }); + m.def("test_callback_pure_virtual_func", + [](VirtClass &virt) { virt.pure_virtual_func(); }); + m.def("test_cross_module_gil", + []() { + auto cm = py::module::import("cross_module_gil_utils"); + auto gil_acquire = reinterpret_cast( + PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); + py::gil_scoped_release gil_release; + gil_acquire(); + }); +} diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_gil_scoped.py spead2-2.1.0/3rdparty/pybind11/tests/test_gil_scoped.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_gil_scoped.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_gil_scoped.py 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,85 @@ +import multiprocessing +import threading +from pybind11_tests import gil_scoped as m + + +def _run_in_process(target, *args, **kwargs): + """Runs target in process and returns its exitcode after 10s (None if still alive).""" + process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) + process.daemon = True + try: + process.start() + # Do not need to wait much, 10s should be more than enough. + process.join(timeout=10) + return process.exitcode + finally: + if process.is_alive(): + process.terminate() + + +def _python_to_cpp_to_python(): + """Calls different C++ functions that come back to Python.""" + class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + extended = ExtendedVirtClass() + m.test_callback_py_obj(lambda: None) + m.test_callback_std_func(lambda: None) + m.test_callback_virtual_func(extended) + m.test_callback_pure_virtual_func(extended) + + +def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): + """Calls different C++ functions that come back to Python, from Python threads.""" + threads = [] + for _ in range(num_threads): + thread = threading.Thread(target=_python_to_cpp_to_python) + thread.daemon = True + thread.start() + if parallel: + threads.append(thread) + else: + thread.join() + for thread in threads: + thread.join() + + +def test_python_to_cpp_to_python_from_thread(): + """Makes sure there is no GIL deadlock when running in a thread. + + It runs in a separate process to be able to stop and assert if it deadlocks. + """ + assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + + +def test_python_to_cpp_to_python_from_thread_multiple_parallel(): + """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. + + It runs in a separate process to be able to stop and assert if it deadlocks. + """ + assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + + +def test_python_to_cpp_to_python_from_thread_multiple_sequential(): + """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. + + It runs in a separate process to be able to stop and assert if it deadlocks. + """ + assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 + + +def test_python_to_cpp_to_python_from_process(): + """Makes sure there is no GIL deadlock when using processes. + + This test is for completion, but it was never an issue. + """ + assert _run_in_process(_python_to_cpp_to_python) == 0 + + +def test_cross_module_gil(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil() # Should not raise a SIGSEGV diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -34,7 +34,9 @@ m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a=0); // test_args_and_kwargs - m.def("args_function", [](py::args args) -> py::tuple { return args; }); + m.def("args_function", [](py::args args) -> py::tuple { + return std::move(args); + }); m.def("args_kwargs_function", [](py::args args, py::kwargs kwargs) { return py::make_tuple(args, kwargs); }); diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.py spead2-2.1.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_kwargs_and_defaults.py 2019-10-08 08:35:40.000000000 +0000 @@ -5,11 +5,11 @@ def test_function_signatures(doc): assert doc(m.kw_func0) == "kw_func0(arg0: int, arg1: int) -> str" assert doc(m.kw_func1) == "kw_func1(x: int, y: int) -> str" - assert doc(m.kw_func2) == "kw_func2(x: int=100, y: int=200) -> str" - assert doc(m.kw_func3) == "kw_func3(data: str='Hello world!') -> None" - assert doc(m.kw_func4) == "kw_func4(myList: List[int]=[13, 17]) -> str" - assert doc(m.kw_func_udl) == "kw_func_udl(x: int, y: int=300) -> str" - assert doc(m.kw_func_udl_z) == "kw_func_udl_z(x: int, y: int=0) -> str" + assert doc(m.kw_func2) == "kw_func2(x: int = 100, y: int = 200) -> str" + assert doc(m.kw_func3) == "kw_func3(data: str = 'Hello world!') -> None" + assert doc(m.kw_func4) == "kw_func4(myList: List[int] = [13, 17]) -> str" + assert doc(m.kw_func_udl) == "kw_func_udl(x: int, y: int = 300) -> str" + assert doc(m.kw_func_udl_z) == "kw_func_udl_z(x: int, y: int = 0) -> str" assert doc(m.args_function) == "args_function(*args) -> tuple" assert doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple" assert doc(m.KWClass.foo0) == \ @@ -93,7 +93,7 @@ assert mpakd(1, i=1) assert msg(excinfo.value) == """ mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: - 1. (i: int=1, j: float=3.14159, *args, **kwargs) -> tuple + 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1; kwargs: i=1 """ # noqa: E501 line too long @@ -101,7 +101,7 @@ assert mpakd(1, 2, j=1) assert msg(excinfo.value) == """ mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported: - 1. (i: int=1, j: float=3.14159, *args, **kwargs) -> tuple + 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1, 2; kwargs: j=1 """ # noqa: E501 line too long diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_local_bindings.py spead2-2.1.0/3rdparty/pybind11/tests/test_local_bindings.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_local_bindings.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_local_bindings.py 2019-10-08 08:35:40.000000000 +0000 @@ -220,7 +220,7 @@ c, d = m.MixGL2(3), cm.MixGL2(4) with pytest.raises(TypeError) as excinfo: m.get_gl_value(c) - assert "incompatible function arguments" in str(excinfo) + assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_gl_value(d) - assert "incompatible function arguments" in str(excinfo) + assert "incompatible function arguments" in str(excinfo.value) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_methods_and_attributes.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_methods_and_attributes.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_methods_and_attributes.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_methods_and_attributes.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -11,6 +11,11 @@ #include "pybind11_tests.h" #include "constructor_stats.h" +#if !defined(PYBIND11_OVERLOAD_CAST) +template +using overload_cast_ = pybind11::detail::overload_cast_impl; +#endif + class ExampleMandA { public: ExampleMandA() { print_default_created(this); } @@ -242,15 +247,16 @@ .def("overloaded_const", py::overload_cast(&ExampleMandA::overloaded, py::const_)) .def("overloaded_const", py::overload_cast(&ExampleMandA::overloaded, py::const_)) #else - .def("overloaded", static_cast(&ExampleMandA::overloaded)) - .def("overloaded", static_cast(&ExampleMandA::overloaded)) - .def("overloaded", static_cast(&ExampleMandA::overloaded)) + // Use both the traditional static_cast method and the C++11 compatible overload_cast_ + .def("overloaded", overload_cast_<>()(&ExampleMandA::overloaded)) + .def("overloaded", overload_cast_()(&ExampleMandA::overloaded)) + .def("overloaded", overload_cast_()(&ExampleMandA::overloaded)) .def("overloaded", static_cast(&ExampleMandA::overloaded)) .def("overloaded", static_cast(&ExampleMandA::overloaded)) .def("overloaded", static_cast(&ExampleMandA::overloaded)) - .def("overloaded_float", static_cast(&ExampleMandA::overloaded)) - .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) - .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) + .def("overloaded_float", overload_cast_()(&ExampleMandA::overloaded)) + .def("overloaded_const", overload_cast_()(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", overload_cast_()(&ExampleMandA::overloaded, py::const_)) .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) @@ -279,12 +285,20 @@ .def(py::init<>()) .def_readonly("def_readonly", &TestProperties::value) .def_readwrite("def_readwrite", &TestProperties::value) + .def_property("def_writeonly", nullptr, + [](TestProperties& s,int v) { s.value = v; } ) + .def_property("def_property_writeonly", nullptr, &TestProperties::set) .def_property_readonly("def_property_readonly", &TestProperties::get) .def_property("def_property", &TestProperties::get, &TestProperties::set) + .def_property("def_property_impossible", nullptr, nullptr) .def_readonly_static("def_readonly_static", &TestProperties::static_value) .def_readwrite_static("def_readwrite_static", &TestProperties::static_value) + .def_property_static("def_writeonly_static", nullptr, + [](py::object, int v) { TestProperties::static_value = v; }) .def_property_readonly_static("def_property_readonly_static", [](py::object) { return TestProperties::static_get(); }) + .def_property_static("def_property_writeonly_static", nullptr, + [](py::object, int v) { return TestProperties::static_set(v); }) .def_property_static("def_property_static", [](py::object) { return TestProperties::static_get(); }, [](py::object, int v) { TestProperties::static_set(v); }) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_methods_and_attributes.py spead2-2.1.0/3rdparty/pybind11/tests/test_methods_and_attributes.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_methods_and_attributes.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_methods_and_attributes.py 2019-10-08 08:35:40.000000000 +0000 @@ -98,23 +98,52 @@ instance.def_property = 3 assert instance.def_property == 3 + with pytest.raises(AttributeError) as excinfo: + dummy = instance.def_property_writeonly # noqa: F841 unused var + assert "unreadable attribute" in str(excinfo.value) + + instance.def_property_writeonly = 4 + assert instance.def_property_readonly == 4 + + with pytest.raises(AttributeError) as excinfo: + dummy = instance.def_property_impossible # noqa: F841 unused var + assert "unreadable attribute" in str(excinfo.value) + + with pytest.raises(AttributeError) as excinfo: + instance.def_property_impossible = 5 + assert "can't set attribute" in str(excinfo.value) + def test_static_properties(): assert m.TestProperties.def_readonly_static == 1 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_readonly_static = 2 - assert "can't set attribute" in str(excinfo) + assert "can't set attribute" in str(excinfo.value) m.TestProperties.def_readwrite_static = 2 assert m.TestProperties.def_readwrite_static == 2 - assert m.TestProperties.def_property_readonly_static == 2 with pytest.raises(AttributeError) as excinfo: - m.TestProperties.def_property_readonly_static = 3 - assert "can't set attribute" in str(excinfo) + dummy = m.TestProperties.def_writeonly_static # noqa: F841 unused var + assert "unreadable attribute" in str(excinfo.value) + + m.TestProperties.def_writeonly_static = 3 + assert m.TestProperties.def_readonly_static == 3 + + assert m.TestProperties.def_property_readonly_static == 3 + with pytest.raises(AttributeError) as excinfo: + m.TestProperties.def_property_readonly_static = 99 + assert "can't set attribute" in str(excinfo.value) + + m.TestProperties.def_property_static = 4 + assert m.TestProperties.def_property_static == 4 + + with pytest.raises(AttributeError) as excinfo: + dummy = m.TestProperties.def_property_writeonly_static + assert "unreadable attribute" in str(excinfo.value) - m.TestProperties.def_property_static = 3 - assert m.TestProperties.def_property_static == 3 + m.TestProperties.def_property_writeonly_static = 5 + assert m.TestProperties.def_property_static == 5 # Static property read and write via instance instance = m.TestProperties() @@ -127,6 +156,13 @@ assert m.TestProperties.def_readwrite_static == 2 assert instance.def_readwrite_static == 2 + with pytest.raises(AttributeError) as excinfo: + dummy = instance.def_property_writeonly_static # noqa: F841 unused var + assert "unreadable attribute" in str(excinfo.value) + + instance.def_property_writeonly_static = 4 + assert instance.def_property_static == 4 + # It should be possible to override properties in derived classes assert m.TestPropertiesOverride().def_readonly == 99 assert m.TestPropertiesOverride.def_readonly_static == 99 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_multiple_inheritance.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_multiple_inheritance.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_multiple_inheritance.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_multiple_inheritance.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -130,8 +130,8 @@ // test_mi_unaligned_base // test_mi_base_return // Issue #801: invalid casting to derived type with MI bases - struct I801B1 { int a = 1; virtual ~I801B1() = default; }; - struct I801B2 { int b = 2; virtual ~I801B2() = default; }; + struct I801B1 { int a = 1; I801B1() = default; I801B1(const I801B1 &) = default; virtual ~I801B1() = default; }; + struct I801B2 { int b = 2; I801B2() = default; I801B2(const I801B2 &) = default; virtual ~I801B2() = default; }; struct I801C : I801B1, I801B2 {}; struct I801D : I801C {}; // Indirect MI // Unregistered classes: @@ -205,7 +205,7 @@ // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance // All of these have int members so that there will be various unequal pointers involved. - struct B { int b; virtual ~B() = default; }; + struct B { int b; B() = default; B(const B&) = default; virtual ~B() = default; }; struct C0 : public virtual B { int c0; }; struct C1 : public virtual B { int c1; }; struct D : public C0, public C1 { int d; }; diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_array.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_array.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_array.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_array.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -14,6 +14,67 @@ #include +// Size / dtype checks. +struct DtypeCheck { + py::dtype numpy{}; + py::dtype pybind11{}; +}; + +template +DtypeCheck get_dtype_check(const char* name) { + py::module np = py::module::import("numpy"); + DtypeCheck check{}; + check.numpy = np.attr("dtype")(np.attr(name)); + check.pybind11 = py::dtype::of(); + return check; +} + +std::vector get_concrete_dtype_checks() { + return { + // Normalization + get_dtype_check("int8"), + get_dtype_check("uint8"), + get_dtype_check("int16"), + get_dtype_check("uint16"), + get_dtype_check("int32"), + get_dtype_check("uint32"), + get_dtype_check("int64"), + get_dtype_check("uint64") + }; +} + +struct DtypeSizeCheck { + std::string name{}; + int size_cpp{}; + int size_numpy{}; + // For debugging. + py::dtype dtype{}; +}; + +template +DtypeSizeCheck get_dtype_size_check() { + DtypeSizeCheck check{}; + check.name = py::type_id(); + check.size_cpp = sizeof(T); + check.dtype = py::dtype::of(); + check.size_numpy = check.dtype.attr("itemsize").template cast(); + return check; +} + +std::vector get_platform_dtype_size_checks() { + return { + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + get_dtype_size_check(), + }; +} + +// Arrays. using arr = py::array; using arr_t = py::array_t; static_assert(std::is_same::value, ""); @@ -68,10 +129,33 @@ return l.release(); } +// note: declaration at local scope would create a dangling reference! +static int data_i = 42; + TEST_SUBMODULE(numpy_array, sm) { try { py::module::import("numpy"); } catch (...) { return; } + // test_dtypes + py::class_(sm, "DtypeCheck") + .def_readonly("numpy", &DtypeCheck::numpy) + .def_readonly("pybind11", &DtypeCheck::pybind11) + .def("__repr__", [](const DtypeCheck& self) { + return py::str("").format( + self.numpy, self.pybind11); + }); + sm.def("get_concrete_dtype_checks", &get_concrete_dtype_checks); + + py::class_(sm, "DtypeSizeCheck") + .def_readonly("name", &DtypeSizeCheck::name) + .def_readonly("size_cpp", &DtypeSizeCheck::size_cpp) + .def_readonly("size_numpy", &DtypeSizeCheck::size_numpy) + .def("__repr__", [](const DtypeSizeCheck& self) { + return py::str("").format( + self.name, self.size_cpp, self.size_numpy, self.dtype); + }); + sm.def("get_platform_dtype_size_checks", &get_platform_dtype_size_checks); + // test_array_attributes sm.def("ndim", [](const arr& a) { return a.ndim(); }); sm.def("shape", [](const arr& a) { return arr(a.ndim(), a.shape()); }); @@ -104,6 +188,8 @@ // test_empty_shaped_array sm.def("make_empty_shaped_array", [] { return py::array(py::dtype("f"), {}, {}); }); + // test numpy scalars (empty shape, ndim==0) + sm.def("scalar_int", []() { return py::array(py::dtype("i"), {}, {}, &data_i); }); // test_wrap sm.def("wrap", [](py::array a) { @@ -295,4 +381,10 @@ std::fill(a.mutable_data(), a.mutable_data() + a.size(), 42.); return a; }); + +#if PY_MAJOR_VERSION >= 3 + sm.def("index_using_ellipsis", [](py::array a) { + return a[py::make_tuple(0, py::ellipsis(), 0)]; + }); +#endif } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_array.py spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_array.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_array.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_array.py 2019-10-08 08:35:40.000000000 +0000 @@ -7,6 +7,21 @@ import numpy as np +def test_dtypes(): + # See issue #1328. + # - Platform-dependent sizes. + for size_check in m.get_platform_dtype_size_checks(): + print(size_check) + assert size_check.size_cpp == size_check.size_numpy, size_check + # - Concrete sizes. + for check in m.get_concrete_dtype_checks(): + print(check) + assert check.numpy == check.pybind11, check + if check.numpy.num != check.pybind11.num: + print("NOTE: typenum mismatch for {}: {} != {}".format( + check, check.numpy.num, check.pybind11.num)) + + @pytest.fixture(scope='function') def arr(): return np.array([[1, 2, 3], [4, 5, 6]], '=u2') @@ -138,6 +153,11 @@ def test_make_empty_shaped_array(): m.make_empty_shaped_array() + # empty shape means numpy scalar, PEP 3118 + assert m.scalar_int().ndim == 0 + assert m.scalar_int().shape == () + assert m.scalar_int() == 42 + def test_wrap(): def assert_references(a, b, base=None): @@ -408,3 +428,20 @@ a = m.create_and_resize(2) assert(a.size == 4) assert(np.all(a == 42.)) + + +@pytest.unsupported_on_py2 +def test_index_using_ellipsis(): + a = m.index_using_ellipsis(np.zeros((5, 6, 7))) + assert a.shape == (6,) + + +@pytest.unsupported_on_pypy +def test_dtype_refcount_leak(): + from sys import getrefcount + dtype = np.dtype(np.float_) + a = np.array([1], dtype=dtype) + before = getrefcount(dtype) + m.ndim(a) + after = getrefcount(dtype) + assert after == before diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_dtypes.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_dtypes.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_dtypes.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_dtypes.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -29,6 +29,13 @@ return os << "s:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } +struct SimpleStructReordered { + bool bool_; + float float_; + uint32_t uint_; + long double ldbl_; +}; + PYBIND11_PACKED(struct PackedStruct { bool bool_; uint32_t uint_; @@ -244,6 +251,9 @@ return list; } +struct A {}; +struct B {}; + TEST_SUBMODULE(numpy_dtypes, m) { try { py::module::import("numpy"); } catch (...) { return; } @@ -252,6 +262,7 @@ py::class_(m, "SimpleStruct"); PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); + PYBIND11_NUMPY_DTYPE(SimpleStructReordered, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(PackedStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); PYBIND11_NUMPY_DTYPE(PartialStruct, bool_, uint_, float_, ldbl_); @@ -271,6 +282,15 @@ // struct NotPOD { std::string v; NotPOD() : v("hi") {}; }; // PYBIND11_NUMPY_DTYPE(NotPOD, v); + // Check that dtypes can be registered programmatically, both from + // initializer lists of field descriptors and from other containers. + py::detail::npy_format_descriptor::register_dtype( + {} + ); + py::detail::npy_format_descriptor::register_dtype( + std::vector{} + ); + // test_recarray, test_scalar_conversion m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); @@ -448,4 +468,7 @@ // test_register_dtype m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); + + // test_str_leak + m.def("dtype_wrapper", [](py::object d) { return py::dtype::from_args(std::move(d)); }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_dtypes.py spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_dtypes.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_numpy_dtypes.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_numpy_dtypes.py 2019-10-08 08:35:40.000000000 +0000 @@ -293,6 +293,18 @@ assert 'dtype is already registered' in str(excinfo.value) -@pytest.requires_numpy +@pytest.unsupported_on_pypy +def test_str_leak(): + from sys import getrefcount + fmt = "f4" + pytest.gc_collect() + start = getrefcount(fmt) + d = m.dtype_wrapper(fmt) + assert d is np.dtype("f4") + del d + pytest.gc_collect() + assert getrefcount(fmt) == start + + def test_compare_buffer_info(): assert all(m.compare_buffer_info()) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_opaque_types.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_opaque_types.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_opaque_types.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_opaque_types.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -11,10 +11,14 @@ #include #include -using StringList = std::vector; +// IMPORTANT: Disable internal pybind11 translation mechanisms for STL data structures +// +// This also deliberately doesn't use the below StringList type alias to test +// that MAKE_OPAQUE can handle a type containing a `,`. (The `std::allocator` +// bit is just the default `std::vector` allocator). +PYBIND11_MAKE_OPAQUE(std::vector>); -/* IMPORTANT: Disable internal pybind11 translation mechanisms for STL data structures */ -PYBIND11_MAKE_OPAQUE(StringList); +using StringList = std::vector>; TEST_SUBMODULE(opaque_types, m) { // test_string_list diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_operator_overloading.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_operator_overloading.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_operator_overloading.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_operator_overloading.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -23,6 +23,7 @@ std::string toString() const { return "[" + std::to_string(x) + ", " + std::to_string(y) + "]"; } + Vector2 operator-() const { return Vector2(-x, -y); } Vector2 operator+(const Vector2 &v) const { return Vector2(x + v.x, y + v.y); } Vector2 operator-(const Vector2 &v) const { return Vector2(x - v.x, y - v.y); } Vector2 operator-(float value) const { return Vector2(x - value, y - value); } @@ -62,6 +63,25 @@ }; } +// MSVC warns about unknown pragmas, and warnings are errors. +#ifndef _MSC_VER + #pragma GCC diagnostic push + // clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to + // `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). + // Here, we suppress the warning using `#pragma diagnostic`. + // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 + // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). + #if (__APPLE__) && (__clang__) + #if (__clang_major__ >= 10) && (__clang_minor__ >= 0) && (__clang_patchlevel__ >= 1) + #pragma GCC diagnostic ignored "-Wself-assign-overloaded" + #endif + #elif (__clang__) + #if (__clang_major__ >= 7) + #pragma GCC diagnostic ignored "-Wself-assign-overloaded" + #endif + #endif +#endif + TEST_SUBMODULE(operators, m) { // test_operator_overloading @@ -85,6 +105,7 @@ .def(float() - py::self) .def(float() * py::self) .def(float() / py::self) + .def(-py::self) .def("__str__", &Vector2::toString) .def(hash(py::self)) ; @@ -144,3 +165,7 @@ .def_readwrite("b", &NestC::b); m.def("get_NestC", [](const NestC &c) { return c.value; }); } + +#ifndef _MSC_VER + #pragma GCC diagnostic pop +#endif diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_operator_overloading.py spead2-2.1.0/3rdparty/pybind11/tests/test_operator_overloading.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_operator_overloading.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_operator_overloading.py 2019-10-08 08:35:40.000000000 +0000 @@ -9,6 +9,8 @@ assert str(v1) == "[1.000000, 2.000000]" assert str(v2) == "[3.000000, -1.000000]" + assert str(-v2) == "[-3.000000, 1.000000]" + assert str(v1 + v2) == "[4.000000, 1.000000]" assert str(v1 - v2) == "[-2.000000, 3.000000]" assert str(v1 - 8) == "[-7.000000, -6.000000]" @@ -44,13 +46,13 @@ del v2 assert cstats.alive() == 0 assert cstats.values() == ['[1.000000, 2.000000]', '[3.000000, -1.000000]', - '[4.000000, 1.000000]', '[-2.000000, 3.000000]', - '[-7.000000, -6.000000]', '[9.000000, 10.000000]', - '[8.000000, 16.000000]', '[0.125000, 0.250000]', - '[7.000000, 6.000000]', '[9.000000, 10.000000]', - '[8.000000, 16.000000]', '[8.000000, 4.000000]', - '[3.000000, -2.000000]', '[3.000000, -0.500000]', - '[6.000000, -2.000000]'] + '[-3.000000, 1.000000]', '[4.000000, 1.000000]', + '[-2.000000, 3.000000]', '[-7.000000, -6.000000]', + '[9.000000, 10.000000]', '[8.000000, 16.000000]', + '[0.125000, 0.250000]', '[7.000000, 6.000000]', + '[9.000000, 10.000000]', '[8.000000, 16.000000]', + '[8.000000, 4.000000]', '[3.000000, -2.000000]', + '[3.000000, -0.500000]', '[6.000000, -2.000000]'] assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 assert cstats.move_constructions >= 10 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_pickling.py spead2-2.1.0/3rdparty/pybind11/tests/test_pickling.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_pickling.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_pickling.py 2019-10-08 08:35:40.000000000 +0000 @@ -34,3 +34,9 @@ assert p2.value == p.value assert p2.extra == p.extra assert p2.dynamic == p.dynamic + + +def test_enum_pickle(): + from pybind11_tests import enums as e + data = pickle.dumps(e.EOne, 2) + assert e.EOne == pickle.loads(data) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_pytypes.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_pytypes.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_pytypes.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_pytypes.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -17,6 +17,8 @@ list.append("value"); py::print("Entry at position 0:", list[0]); list[0] = py::str("overwritten"); + list.insert(0, "inserted-0"); + list.insert(2, "inserted-2"); return list; }); m.def("print_list", [](py::list list) { @@ -37,6 +39,12 @@ for (auto item : set) py::print("key:", item); }); + m.def("set_contains", [](py::set set, py::object key) { + return set.contains(key); + }); + m.def("set_contains", [](py::set set, const char* key) { + return set.contains(key); + }); // test_dict m.def("get_dict", []() { return py::dict("key"_a="value"); }); @@ -49,6 +57,12 @@ auto d2 = py::dict("z"_a=3, **d1); return d2; }); + m.def("dict_contains", [](py::dict dict, py::object val) { + return dict.contains(val); + }); + m.def("dict_contains", [](py::dict dict, const char* val) { + return dict.contains(val); + }); // test_str m.def("str_from_string", []() { return py::str(std::string("baz")); }); @@ -269,4 +283,28 @@ m.def("print_failure", []() { py::print(42, UnregisteredType()); }); m.def("hash_function", [](py::object obj) { return py::hash(obj); }); + + m.def("test_number_protocol", [](py::object a, py::object b) { + py::list l; + l.append(a.equal(b)); + l.append(a.not_equal(b)); + l.append(a < b); + l.append(a <= b); + l.append(a > b); + l.append(a >= b); + l.append(a + b); + l.append(a - b); + l.append(a * b); + l.append(a / b); + l.append(a | b); + l.append(a & b); + l.append(a ^ b); + l.append(a >> b); + l.append(a << b); + return l; + }); + + m.def("test_list_slicing", [](py::list a) { + return a[py::slice(0, -1, 2)]; + }); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_pytypes.py spead2-2.1.0/3rdparty/pybind11/tests/test_pytypes.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_pytypes.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_pytypes.py 2019-10-08 08:35:40.000000000 +0000 @@ -1,3 +1,4 @@ +from __future__ import division import pytest import sys @@ -8,14 +9,16 @@ def test_list(capture, doc): with capture: lst = m.get_list() - assert lst == ["overwritten"] + assert lst == ["inserted-0", "overwritten", "inserted-2"] lst.append("value2") m.print_list(lst) assert capture.unordered == """ Entry at position 0: value - list item 0: overwritten - list item 1: value2 + list item 0: inserted-0 + list item 1: overwritten + list item 2: inserted-2 + list item 3: value2 """ assert doc(m.get_list) == "get_list() -> list" @@ -36,6 +39,10 @@ key: key4 """ + assert not m.set_contains(set([]), 42) + assert m.set_contains({42}, 42) + assert m.set_contains({"foo"}, "foo") + assert doc(m.get_list) == "get_list() -> list" assert doc(m.print_list) == "print_list(arg0: list) -> None" @@ -52,6 +59,10 @@ key: key2, value=value2 """ + assert not m.dict_contains({}, 42) + assert m.dict_contains({42: None}, 42) + assert m.dict_contains({"foo": None}, "foo") + assert doc(m.get_dict) == "get_dict() -> dict" assert doc(m.print_dict) == "print_dict(arg0: dict) -> None" @@ -238,3 +249,15 @@ assert m.hash_function(Hashable(42)) == 42 with pytest.raises(TypeError): m.hash_function(Unhashable()) + + +def test_number_protocol(): + for a, b in [(1, 1), (3, 5)]: + li = [a == b, a != b, a < b, a <= b, a > b, a >= b, a + b, + a - b, a * b, a / b, a | b, a & b, a ^ b, a >> b, a << b] + assert m.test_number_protocol(a, b) == li + + +def test_list_slicing(): + li = list(range(100)) + assert li[::2] == m.test_list_slicing(li) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_sequences_and_iterators.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_sequences_and_iterators.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_sequences_and_iterators.cpp 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_sequences_and_iterators.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -71,6 +71,25 @@ } TEST_SUBMODULE(sequences_and_iterators, m) { + // test_sliceable + class Sliceable{ + public: + Sliceable(int n): size(n) {} + int start,stop,step; + int size; + }; + py::class_(m,"Sliceable") + .def(py::init()) + .def("__getitem__",[](const Sliceable &s, py::slice slice) { + ssize_t start, stop, step, slicelength; + if (!slice.compute(s.size, &start, &stop, &step, &slicelength)) + throw py::error_already_set(); + int istart = static_cast(start); + int istop = static_cast(stop); + int istep = static_cast(step); + return std::make_tuple(istart,istop,istep); + }) + ; // test_sequence class Sequence { diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_sequences_and_iterators.py spead2-2.1.0/3rdparty/pybind11/tests/test_sequences_and_iterators.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_sequences_and_iterators.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_sequences_and_iterators.py 2019-10-08 08:35:40.000000000 +0000 @@ -33,6 +33,19 @@ next(it) +def test_sliceable(): + sliceable = m.Sliceable(100) + assert sliceable[::] == (0, 100, 1) + assert sliceable[10::] == (10, 100, 1) + assert sliceable[:10:] == (0, 10, 1) + assert sliceable[::10] == (0, 100, 10) + assert sliceable[-10::] == (90, 100, 1) + assert sliceable[:-10:] == (0, 90, 1) + assert sliceable[::-10] == (99, -1, -10) + assert sliceable[50:60:1] == (50, 60, 1) + assert sliceable[50:60:-1] == (50, 60, -1) + + def test_sequence(): cstats = ConstructorStats.get(m.Sequence) diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_smart_ptr.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_smart_ptr.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_smart_ptr.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_smart_ptr.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -127,6 +127,7 @@ // Object managed by a std::shared_ptr<> class MyObject2 { public: + MyObject2(const MyObject2 &) = default; MyObject2(int value) : value(value) { print_created(this, toString()); } std::string toString() const { return "MyObject2[" + std::to_string(value) + "]"; } virtual ~MyObject2() { print_destroyed(this); } @@ -145,6 +146,7 @@ // Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<> class MyObject3 : public std::enable_shared_from_this { public: + MyObject3(const MyObject3 &) = default; MyObject3(int value) : value(value) { print_created(this, toString()); } std::string toString() const { return "MyObject3[" + std::to_string(value) + "]"; } virtual ~MyObject3() { print_destroyed(this); } @@ -184,6 +186,32 @@ .def(py::init()) .def_readwrite("value", &MyObject4::value); + // test_unique_deleter + // Object with std::unique_ptr where D is not matching the base class + // Object with a protected destructor + class MyObject4a { + public: + MyObject4a(int i) { + value = i; + print_created(this); + }; + int value; + protected: + virtual ~MyObject4a() { print_destroyed(this); } + }; + py::class_>(m, "MyObject4a") + .def(py::init()) + .def_readwrite("value", &MyObject4a::value); + + // Object derived but with public destructor and no Deleter in default holder + class MyObject4b : public MyObject4a { + public: + MyObject4b(int i) : MyObject4a(i) { print_created(this); } + ~MyObject4b() { print_destroyed(this); } + }; + py::class_(m, "MyObject4b") + .def(py::init()); + // test_large_holder class MyObject5 { // managed by huge_unique_ptr public: @@ -248,6 +276,8 @@ // Issue #865: shared_from_this doesn't work with virtual inheritance struct SharedFromThisVBase : std::enable_shared_from_this { + SharedFromThisVBase() = default; + SharedFromThisVBase(const SharedFromThisVBase &) = default; virtual ~SharedFromThisVBase() = default; }; struct SharedFromThisVirt : virtual SharedFromThisVBase {}; @@ -306,7 +336,9 @@ // test_shared_ptr_gc // #187: issue involving std::shared_ptr<> return value policy & garbage collection - struct ElementBase { virtual void foo() { } /* Force creation of virtual table */ }; + struct ElementBase { + virtual ~ElementBase() { } /* Force creation of virtual table */ + }; py::class_>(m, "ElementBase"); struct ElementA : ElementBase { diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_smart_ptr.py spead2-2.1.0/3rdparty/pybind11/tests/test_smart_ptr.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_smart_ptr.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_smart_ptr.py 2019-10-08 08:35:40.000000000 +0000 @@ -115,6 +115,27 @@ assert cstats.alive() == 1 # Leak, but that's intentional +def test_unique_nodelete4a(): + o = m.MyObject4a(23) + assert o.value == 23 + cstats = ConstructorStats.get(m.MyObject4a) + assert cstats.alive() == 1 + del o + assert cstats.alive() == 1 # Leak, but that's intentional + + +def test_unique_deleter(): + o = m.MyObject4b(23) + assert o.value == 23 + cstats4a = ConstructorStats.get(m.MyObject4a) + assert cstats4a.alive() == 2 # Two because of previous test + cstats4b = ConstructorStats.get(m.MyObject4b) + assert cstats4b.alive() == 1 + del o + assert cstats4a.alive() == 1 # Should now only be one leftover from previous test + assert cstats4b.alive() == 0 # Should be deleted + + def test_large_holder(): o = m.MyObject5(5) assert o.value == 5 @@ -251,7 +272,8 @@ instance = m.HeldByDefaultHolder() with pytest.raises(RuntimeError) as excinfo: m.HeldByDefaultHolder.load_shared_ptr(instance) - assert "Unable to load a custom holder type from a default-holder instance" in str(excinfo) + assert "Unable to load a custom holder type from a " \ + "default-holder instance" in str(excinfo.value) def test_shared_ptr_gc(): diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_stl_binders.py spead2-2.1.0/3rdparty/pybind11/tests/test_stl_binders.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_stl_binders.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_stl_binders.py 2019-10-08 08:35:40.000000000 +0000 @@ -11,6 +11,10 @@ assert len(v_int) == 2 assert bool(v_int) is True + # test construction from a generator + v_int1 = m.VectorInt(x for x in range(5)) + assert v_int1 == m.VectorInt([0, 1, 2, 3, 4]) + v_int2 = m.VectorInt([0, 0]) assert v_int == v_int2 v_int2[1] = 1 @@ -33,6 +37,32 @@ del v_int2[0] assert v_int2 == m.VectorInt([0, 99, 2, 3]) + v_int2.extend(m.VectorInt([4, 5])) + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5]) + + v_int2.extend([6, 7]) + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5, 6, 7]) + + # test error handling, and that the vector is unchanged + with pytest.raises(RuntimeError): + v_int2.extend([8, 'a']) + + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5, 6, 7]) + + # test extending from a generator + v_int2.extend(x for x in range(5)) + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4]) + + # test negative indexing + assert v_int2[-1] == 4 + + # insert with negative index + v_int2.insert(-1, 88) + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 88, 4]) + + # delete negative index + del v_int2[-1] + assert v_int2 == m.VectorInt([0, 99, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 88]) # related to the PyPy's buffer protocol. @pytest.unsupported_on_pypy diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_stl.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_stl.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_stl.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_stl.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -11,6 +11,9 @@ #include "constructor_stats.h" #include +#include +#include + // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 #if PYBIND11_HAS_VARIANT using std::variant; @@ -33,6 +36,8 @@ }} // namespace pybind11::detail #endif +PYBIND11_MAKE_OPAQUE(std::vector>); + /// Issue #528: templated constructor struct TplCtorClass { template TplCtorClass(const T &) { } @@ -58,6 +63,10 @@ static std::vector lvv{2}; m.def("cast_ptr_vector", []() { return &lvv; }); + // test_deque + m.def("cast_deque", []() { return std::deque{1}; }); + m.def("load_deque", [](const std::deque &v) { return v.at(0) == 1 && v.at(1) == 2; }); + // test_array m.def("cast_array", []() { return std::array {{1 , 2}}; }); m.def("load_array", [](const std::array &a) { return a[0] == 1 && a[1] == 2; }); @@ -237,6 +246,11 @@ // test_stl_pass_by_pointer m.def("stl_pass_by_pointer", [](std::vector* v) { return *v; }, "v"_a=nullptr); + // #1258: pybind11/stl.h converts string to vector + m.def("func_with_string_or_vector_string_arg_overload", [](std::vector) { return 1; }); + m.def("func_with_string_or_vector_string_arg_overload", [](std::list) { return 2; }); + m.def("func_with_string_or_vector_string_arg_overload", [](std::string) { return 3; }); + class Placeholder { public: Placeholder() { print_created(this); } @@ -253,4 +267,18 @@ return result; }, py::return_value_policy::take_ownership); + + m.def("array_cast_sequence", [](std::array x) { return x; }); + + /// test_issue_1561 + struct Issue1561Inner { std::string data; }; + struct Issue1561Outer { std::vector list; }; + + py::class_(m, "Issue1561Inner") + .def(py::init()) + .def_readwrite("data", &Issue1561Inner::data); + + py::class_(m, "Issue1561Outer") + .def(py::init<>()) + .def_readwrite("list", &Issue1561Outer::list); } diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_stl.py spead2-2.1.0/3rdparty/pybind11/tests/test_stl.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_stl.py 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_stl.py 2019-10-08 08:35:40.000000000 +0000 @@ -23,6 +23,15 @@ assert m.cast_ptr_vector() == ["lvalue", "lvalue"] +def test_deque(doc): + """std::deque <-> list""" + lst = m.cast_deque() + assert lst == [1] + lst.append(2) + assert m.load_deque(lst) + assert m.load_deque(tuple(lst)) + + def test_array(doc): """std::array <-> list""" lst = m.cast_array() @@ -47,7 +56,9 @@ """std::map <-> dict""" d = m.cast_map() assert d == {"key": "value"} + assert "key" in d d["key2"] = "value2" + assert "key2" in d assert m.load_map(d) assert doc(m.cast_map) == "cast_map() -> Dict[str, str]" @@ -165,7 +176,7 @@ m.stl_pass_by_pointer() # default value is `nullptr` assert msg(excinfo.value) == """ stl_pass_by_pointer(): incompatible function arguments. The following argument types are supported: - 1. (v: List[int]=None) -> List[int] + 1. (v: List[int] = None) -> List[int] Invoked with: """ # noqa: E501 line too long @@ -174,7 +185,7 @@ m.stl_pass_by_pointer(None) assert msg(excinfo.value) == """ stl_pass_by_pointer(): incompatible function arguments. The following argument types are supported: - 1. (v: List[int]=None) -> List[int] + 1. (v: List[int] = None) -> List[int] Invoked with: None """ # noqa: E501 line too long @@ -201,6 +212,14 @@ assert expected_message in str(excinfo.value) +def test_function_with_string_and_vector_string_arg(): + """Check if a string is NOT implicitly converted to a list, which was the + behavior before fix of issue #1258""" + assert m.func_with_string_or_vector_string_arg_overload(('A', 'B', )) == 2 + assert m.func_with_string_or_vector_string_arg_overload(['A', 'B']) == 2 + assert m.func_with_string_or_vector_string_arg_overload('A') == 3 + + def test_stl_ownership(): cstats = ConstructorStats.get(m.Placeholder) assert cstats.alive() == 0 @@ -208,3 +227,15 @@ assert len(r) == 1 del r assert cstats.alive() == 0 + + +def test_array_cast_sequence(): + assert m.array_cast_sequence((1, 2, 3)) == [1, 2, 3] + + +def test_issue_1561(): + """ check fix for issue #1561 """ + bar = m.Issue1561Outer() + bar.list = [m.Issue1561Inner('bar')] + bar.list + assert bar.list[0].data == 'bar' diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,136 @@ +/* + tests/test_tagbased_polymorphic.cpp -- test of polymorphic_type_hook + + Copyright (c) 2018 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "pybind11_tests.h" +#include + +struct Animal +{ + enum class Kind { + Unknown = 0, + Dog = 100, Labrador, Chihuahua, LastDog = 199, + Cat = 200, Panther, LastCat = 299 + }; + static const std::type_info* type_of_kind(Kind kind); + static std::string name_of_kind(Kind kind); + + const Kind kind; + const std::string name; + + protected: + Animal(const std::string& _name, Kind _kind) + : kind(_kind), name(_name) + {} +}; + +struct Dog : Animal +{ + Dog(const std::string& _name, Kind _kind = Kind::Dog) : Animal(_name, _kind) {} + std::string bark() const { return name_of_kind(kind) + " " + name + " goes " + sound; } + std::string sound = "WOOF!"; +}; + +struct Labrador : Dog +{ + Labrador(const std::string& _name, int _excitement = 9001) + : Dog(_name, Kind::Labrador), excitement(_excitement) {} + int excitement; +}; + +struct Chihuahua : Dog +{ + Chihuahua(const std::string& _name) : Dog(_name, Kind::Chihuahua) { sound = "iyiyiyiyiyi"; } + std::string bark() const { return Dog::bark() + " and runs in circles"; } +}; + +struct Cat : Animal +{ + Cat(const std::string& _name, Kind _kind = Kind::Cat) : Animal(_name, _kind) {} + std::string purr() const { return "mrowr"; } +}; + +struct Panther : Cat +{ + Panther(const std::string& _name) : Cat(_name, Kind::Panther) {} + std::string purr() const { return "mrrrRRRRRR"; } +}; + +std::vector> create_zoo() +{ + std::vector> ret; + ret.emplace_back(new Labrador("Fido", 15000)); + + // simulate some new type of Dog that the Python bindings + // haven't been updated for; it should still be considered + // a Dog, not just an Animal. + ret.emplace_back(new Dog("Ginger", Dog::Kind(150))); + + ret.emplace_back(new Chihuahua("Hertzl")); + ret.emplace_back(new Cat("Tiger", Cat::Kind::Cat)); + ret.emplace_back(new Panther("Leo")); + return ret; +} + +const std::type_info* Animal::type_of_kind(Kind kind) +{ + switch (kind) { + case Kind::Unknown: break; + + case Kind::Dog: break; + case Kind::Labrador: return &typeid(Labrador); + case Kind::Chihuahua: return &typeid(Chihuahua); + case Kind::LastDog: break; + + case Kind::Cat: break; + case Kind::Panther: return &typeid(Panther); + case Kind::LastCat: break; + } + + if (kind >= Kind::Dog && kind <= Kind::LastDog) return &typeid(Dog); + if (kind >= Kind::Cat && kind <= Kind::LastCat) return &typeid(Cat); + return nullptr; +} + +std::string Animal::name_of_kind(Kind kind) +{ + std::string raw_name = type_of_kind(kind)->name(); + py::detail::clean_type_id(raw_name); + return raw_name; +} + +namespace pybind11 { + template + struct polymorphic_type_hook::value>> + { + static const void *get(const itype *src, const std::type_info*& type) + { type = src ? Animal::type_of_kind(src->kind) : nullptr; return src; } + }; +} + +TEST_SUBMODULE(tagbased_polymorphic, m) { + py::class_(m, "Animal") + .def_readonly("name", &Animal::name); + py::class_(m, "Dog") + .def(py::init()) + .def_readwrite("sound", &Dog::sound) + .def("bark", &Dog::bark); + py::class_(m, "Labrador") + .def(py::init(), "name"_a, "excitement"_a = 9001) + .def_readwrite("excitement", &Labrador::excitement); + py::class_(m, "Chihuahua") + .def(py::init()) + .def("bark", &Chihuahua::bark); + py::class_(m, "Cat") + .def(py::init()) + .def("purr", &Cat::purr); + py::class_(m, "Panther") + .def(py::init()) + .def("purr", &Panther::purr); + m.def("create_zoo", &create_zoo); +}; diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.py spead2-2.1.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_tagbased_polymorphic.py 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,20 @@ +from pybind11_tests import tagbased_polymorphic as m + + +def test_downcast(): + zoo = m.create_zoo() + assert [type(animal) for animal in zoo] == [ + m.Labrador, m.Dog, m.Chihuahua, m.Cat, m.Panther + ] + assert [animal.name for animal in zoo] == [ + "Fido", "Ginger", "Hertzl", "Tiger", "Leo" + ] + zoo[1].sound = "woooooo" + assert [dog.bark() for dog in zoo[:3]] == [ + "Labrador Fido goes WOOF!", + "Dog Ginger goes woooooo", + "Chihuahua Hertzl goes iyiyiyiyiyi and runs in circles" + ] + assert [cat.purr() for cat in zoo[3:]] == ["mrowr", "mrrrRRRRRR"] + zoo[0].excitement -= 1000 + assert zoo[0].excitement == 14000 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_union.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_union.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_union.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_union.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,22 @@ +/* + tests/test_class.cpp -- test py::class_ definitions and basic functionality + + Copyright (c) 2019 Roland Dreier + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "pybind11_tests.h" + +TEST_SUBMODULE(union_, m) { + union TestUnion { + int value_int; + unsigned value_uint; + }; + + py::class_(m, "TestUnion") + .def(py::init<>()) + .def_readonly("as_int", &TestUnion::value_int) + .def_readwrite("as_uint", &TestUnion::value_uint); +} diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_union.py spead2-2.1.0/3rdparty/pybind11/tests/test_union.py --- spead2-1.10.0/3rdparty/pybind11/tests/test_union.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_union.py 2019-10-08 08:35:40.000000000 +0000 @@ -0,0 +1,8 @@ +from pybind11_tests import union_ as m + + +def test_union(): + instance = m.TestUnion() + + instance.as_uint = 10 + assert instance.as_int == 10 diff -Nru spead2-1.10.0/3rdparty/pybind11/tests/test_virtual_functions.cpp spead2-2.1.0/3rdparty/pybind11/tests/test_virtual_functions.cpp --- spead2-1.10.0/3rdparty/pybind11/tests/test_virtual_functions.cpp 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tests/test_virtual_functions.cpp 2019-10-08 08:35:40.000000000 +0000 @@ -129,6 +129,7 @@ class NCVirt { public: + virtual ~NCVirt() { } virtual NonCopyable get_noncopyable(int a, int b) { return NonCopyable(a, b); } virtual Movable get_movable(int a, int b) = 0; @@ -230,7 +231,9 @@ void f() override { py::print("PyA.f()"); - PYBIND11_OVERLOAD(void, A, f); + // This convolution just gives a `void`, but tests that PYBIND11_TYPE() works to protect + // a type containing a , + PYBIND11_OVERLOAD(PYBIND11_TYPE(typename std::enable_if::type), A, f); } }; diff -Nru spead2-1.10.0/3rdparty/pybind11/tools/FindPythonLibsNew.cmake spead2-2.1.0/3rdparty/pybind11/tools/FindPythonLibsNew.cmake --- spead2-1.10.0/3rdparty/pybind11/tools/FindPythonLibsNew.cmake 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tools/FindPythonLibsNew.cmake 2019-10-08 08:35:40.000000000 +0000 @@ -64,6 +64,7 @@ if(NOT PYTHONINTERP_FOUND) set(PYTHONLIBS_FOUND FALSE) + set(PythonLibsNew_FOUND FALSE) return() endif() @@ -96,10 +97,14 @@ "Python config failure:\n${_PYTHON_ERROR_VALUE}") endif() set(PYTHONLIBS_FOUND FALSE) + set(PythonLibsNew_FOUND FALSE) return() endif() # Convert the process output into a list +if(WIN32) + string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) +endif() string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) @@ -124,6 +129,7 @@ "chosen compiler is ${_CMAKE_BITS}-bit") endif() set(PYTHONLIBS_FOUND FALSE) + set(PythonLibsNew_FOUND FALSE) return() endif() @@ -138,7 +144,7 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR}) string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES}) -if(CMAKE_HOST_WIN32) +if(CMAKE_HOST_WIN32 AND NOT (MSYS OR MINGW)) set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib") @@ -193,3 +199,4 @@ "${PYTHON_EXECUTABLE}${PYTHON_VERSION}") set(PYTHONLIBS_FOUND TRUE) +set(PythonLibsNew_FOUND TRUE) diff -Nru spead2-1.10.0/3rdparty/pybind11/tools/mkdoc.py spead2-2.1.0/3rdparty/pybind11/tools/mkdoc.py --- spead2-1.10.0/3rdparty/pybind11/tools/mkdoc.py 2018-08-31 07:33:43.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tools/mkdoc.py 2019-10-08 08:35:40.000000000 +0000 @@ -14,6 +14,7 @@ from clang import cindex from clang.cindex import CursorKind from collections import OrderedDict +from glob import glob from threading import Thread, Semaphore from multiprocessing import cpu_count @@ -40,6 +41,10 @@ CursorKind.FIELD_DECL ] +PREFIX_BLACKLIST = [ + CursorKind.TRANSLATION_UNIT +] + CPP_OPERATORS = { '<=': 'le', '>=': 'ge', '==': 'eq', '!=': 'ne', '[]': 'array', '+=': 'iadd', '-=': 'isub', '*=': 'imul', '/=': 'idiv', '%=': @@ -56,10 +61,13 @@ job_count = cpu_count() job_semaphore = Semaphore(job_count) -output = [] + +class NoFilenamesError(ValueError): + pass + def d(s): - return s.decode('utf8') + return s if isinstance(s, str) else s.decode('utf8') def sanitize_name(name): @@ -182,18 +190,18 @@ return result.rstrip().lstrip('\n') -def extract(filename, node, prefix): +def extract(filename, node, prefix, output): if not (node.location.file is None or os.path.samefile(d(node.location.file.name), filename)): return 0 if node.kind in RECURSE_LIST: sub_prefix = prefix - if node.kind != CursorKind.TRANSLATION_UNIT: + if node.kind not in PREFIX_BLACKLIST: if len(sub_prefix) > 0: sub_prefix += '_' sub_prefix += d(node.spelling) for i in node.get_children(): - extract(filename, i, sub_prefix) + extract(filename, i, sub_prefix, output) if node.kind in PRINT_LIST: comment = d(node.raw_comment) if node.raw_comment is not None else '' comment = process_comment(comment) @@ -202,15 +210,15 @@ sub_prefix += '_' if len(node.spelling) > 0: name = sanitize_name(sub_prefix + d(node.spelling)) - global output output.append((name, filename, comment)) class ExtractionThread(Thread): - def __init__(self, filename, parameters): + def __init__(self, filename, parameters, output): Thread.__init__(self) self.filename = filename self.parameters = parameters + self.output = output job_semaphore.acquire() def run(self): @@ -219,13 +227,18 @@ index = cindex.Index( cindex.conf.lib.clang_createIndex(False, True)) tu = index.parse(self.filename, self.parameters) - extract(self.filename, tu.cursor, '') + extract(self.filename, tu.cursor, '', self.output) finally: job_semaphore.release() -if __name__ == '__main__': - parameters = ['-x', 'c++', '-std=c++11'] + +def read_args(args): + parameters = [] filenames = [] + if "-x" not in args: + parameters.extend(['-x', 'c++']) + if not any(it.startswith("-std=") for it in args): + parameters.append('-std=c++11') if platform.system() == 'Darwin': dev_path = '/Applications/Xcode.app/Contents/Developer/' @@ -240,17 +253,48 @@ sysroot_dir = os.path.join(sdk_dir, next(os.walk(sdk_dir))[1][0]) parameters.append('-isysroot') parameters.append(sysroot_dir) + elif platform.system() == 'Linux': + # clang doesn't find its own base includes by default on Linux, + # but different distros install them in different paths. + # Try to autodetect, preferring the highest numbered version. + def clang_folder_version(d): + return [int(ver) for ver in re.findall(r'(?:${PYBIND11_CPP_STANDARD}>) + endif() get_property(_iid TARGET ${PN}::pybind11 PROPERTY INTERFACE_INCLUDE_DIRECTORIES) get_property(_ill TARGET ${PN}::module PROPERTY INTERFACE_LINK_LIBRARIES) diff -Nru spead2-1.10.0/3rdparty/pybind11/tools/pybind11Tools.cmake spead2-2.1.0/3rdparty/pybind11/tools/pybind11Tools.cmake --- spead2-1.10.0/3rdparty/pybind11/tools/pybind11Tools.cmake 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/tools/pybind11Tools.cmake 2019-10-08 08:35:40.000000000 +0000 @@ -110,10 +110,10 @@ # Build a Python extension module: # pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] -# [NO_EXTRAS] [THIN_LTO] source1 [source2 ...]) +# [NO_EXTRAS] [SYSTEM] [THIN_LTO] source1 [source2 ...]) # function(pybind11_add_module target_name) - set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS THIN_LTO) + set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO) cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) if(ARG_MODULE AND ARG_SHARED) @@ -130,7 +130,11 @@ add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS}) - target_include_directories(${target_name} + if(ARG_SYSTEM) + set(inc_isystem SYSTEM) + endif() + + target_include_directories(${target_name} ${inc_isystem} PRIVATE ${PYBIND11_INCLUDE_DIR} # from project CMakeLists.txt PRIVATE ${pybind11_INCLUDE_DIR} # from pybind11Config PRIVATE ${PYTHON_INCLUDE_DIRS}) @@ -152,6 +156,7 @@ # namespace; also turning it on for a pybind module compilation here avoids # potential warnings or issues from having mixed hidden/non-hidden types. set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden") + set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden") if(WIN32 OR CYGWIN) # Link against the Python shared library on Windows @@ -180,7 +185,11 @@ endif() # Make sure C++11/14 are enabled - target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD}) + if(CMAKE_VERSION VERSION_LESS 3.3) + target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD}) + else() + target_compile_options(${target_name} PUBLIC $<$:${PYBIND11_CPP_STANDARD}>) + endif() if(ARG_NO_EXTRAS) return() @@ -188,7 +197,7 @@ _pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO}) - if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) + if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) # Strip unnecessary sections of the binary on Linux/Mac OS if(CMAKE_STRIP) if(APPLE) @@ -204,6 +213,15 @@ if(MSVC) # /MP enables multithreaded builds (relevant when there are many files), /bigobj is # needed for bigger binding projects due to the limit to 64k addressable sections - target_compile_options(${target_name} PRIVATE /MP /bigobj) + target_compile_options(${target_name} PRIVATE /bigobj) + if(CMAKE_VERSION VERSION_LESS 3.11) + target_compile_options(${target_name} PRIVATE $<$>:/MP>) + else() + # Only set these options for C++ files. This is important so that, for + # instance, projects that include other types of source files like CUDA + # .cu files don't get these options propagated to nvcc since that would + # cause the build to fail. + target_compile_options(${target_name} PRIVATE $<$>:$<$:/MP>>) + endif() endif() endfunction() diff -Nru spead2-1.10.0/3rdparty/pybind11/.travis.yml spead2-2.1.0/3rdparty/pybind11/.travis.yml --- spead2-1.10.0/3rdparty/pybind11/.travis.yml 2018-10-15 08:56:32.000000000 +0000 +++ spead2-2.1.0/3rdparty/pybind11/.travis.yml 2019-10-08 08:35:40.000000000 +0000 @@ -1,6 +1,4 @@ language: cpp -dist: trusty -sudo: false matrix: include: # This config does a few things: @@ -10,16 +8,17 @@ # - Makes sure that everything still works without optional deps (numpy/scipy/eigen) and # also tests the automatic discovery functions in CMake (Python version, C++ standard). - os: linux - env: STYLE DOCS PIP + dist: xenial # Necessary to run doxygen 1.8.15 + name: Style, docs, and pip cache: false before_install: - pyenv global $(pyenv whence 2to3) # activate all python versions - PY_CMD=python3 - - $PY_CMD -m pip install --user --upgrade pip wheel + - $PY_CMD -m pip install --user --upgrade pip wheel setuptools install: - $PY_CMD -m pip install --user --upgrade sphinx sphinx_rtd_theme breathe flake8 pep8-naming pytest - - curl -fsSL ftp://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.12.linux.bin.tar.gz | tar xz - - export PATH="$PWD/doxygen-1.8.12/bin:$PATH" + - curl -fsSL https://sourceforge.net/projects/doxygen/files/rel-1.8.15/doxygen-1.8.15.linux.bin.tar.gz/download | tar xz + - export PATH="$PWD/doxygen-1.8.15/bin:$PATH" script: - tools/check-style.sh - flake8 @@ -32,62 +31,119 @@ diff -rq $installed ./include/pybind11 - | # Barebones build - cmake -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + cmake -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DPYTHON_EXECUTABLE=$(which $PY_CMD) . make pytest -j 2 make cpptest -j 2 # The following are regular test configurations, including optional dependencies. # With regard to each other they differ in Python version, C++ standard and compiler. - os: linux + dist: trusty + name: Python 2.7, c++11, gcc 4.8 env: PYTHON=2.7 CPP=11 GCC=4.8 addons: apt: - packages: [cmake=2.\*, cmake-data=2.\*] + packages: + - cmake=2.\* + - cmake-data=2.\* - os: linux + dist: trusty + name: Python 3.6, c++11, gcc 4.8 env: PYTHON=3.6 CPP=11 GCC=4.8 addons: apt: - sources: [deadsnakes] - packages: [python3.6-dev python3.6-venv, cmake=2.\*, cmake-data=2.\*] - - sudo: true - services: docker + sources: + - deadsnakes + packages: + - python3.6-dev + - python3.6-venv + - cmake=2.\* + - cmake-data=2.\* + - os: linux + dist: trusty env: PYTHON=2.7 CPP=14 GCC=6 CMAKE=1 - - sudo: true - services: docker - env: PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 - - sudo: true + name: Python 2.7, c++14, gcc 4.8, CMake test + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-6 + - os: linux + dist: trusty + name: Python 3.5, c++14, gcc 6, Debug build + # N.B. `ensurepip` could be installed transitively by `python3.5-venv`, but + # seems to have apt conflicts (at least for Trusty). Use Docker instead. services: docker + env: DOCKER=debian:stretch PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 + - os: linux + dist: xenial env: PYTHON=3.6 CPP=17 GCC=7 + name: Python 3.6, c++17, gcc 7 + addons: + apt: + sources: + - deadsnakes + - ubuntu-toolchain-r-test + packages: + - g++-7 + - python3.6-dev + - python3.6-venv - os: linux - env: PYTHON=3.6 CPP=17 CLANG=5.0 + dist: xenial + env: PYTHON=3.6 CPP=17 CLANG=7 + name: Python 3.6, c++17, Clang 7 addons: apt: - sources: [deadsnakes, llvm-toolchain-trusty-5.0, ubuntu-toolchain-r-test] - packages: [python3.6-dev python3.6-venv clang-5.0 llvm-5.0-dev, lld-5.0] + sources: + - deadsnakes + - llvm-toolchain-xenial-7 + packages: + - python3.6-dev + - python3.6-venv + - clang-7 + - libclang-7-dev + - llvm-7-dev + - lld-7 + - libc++-7-dev + - libc++abi-7-dev # Why is this necessary??? - os: osx + name: Python 2.7, c++14, AppleClang 7.3, CMake test osx_image: xcode7.3 env: PYTHON=2.7 CPP=14 CLANG CMAKE=1 - os: osx + name: Python 3.7, c++14, AppleClang 9, Debug build osx_image: xcode9 env: PYTHON=3.7 CPP=14 CLANG DEBUG=1 # Test a PyPy 2.7 build - os: linux + dist: trusty env: PYPY=5.8 PYTHON=2.7 CPP=11 GCC=4.8 + name: PyPy 5.8, Python 2.7, c++11, gcc 4.8 addons: apt: - packages: [libblas-dev, liblapack-dev, gfortran] + packages: + - libblas-dev + - liblapack-dev + - gfortran # Build in 32-bit mode and tests against the CMake-installed version - - sudo: true + - os: linux + dist: trusty services: docker - env: ARCH=i386 PYTHON=3.5 CPP=14 GCC=6 INSTALL=1 + env: DOCKER=i386/debian:stretch PYTHON=3.5 CPP=14 GCC=6 INSTALL=1 + name: Python 3.4, c++14, gcc 6, 32-bit script: - | - $SCRIPT_RUN_PREFIX sh -c "set -e - cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 - make install - cp -a tests /pybind11-tests - mkdir /build-tests && cd /build-tests - cmake ../pybind11-tests ${CMAKE_EXTRA_ARGS} -DPYBIND11_WERROR=ON - make pytest -j 2" + # Consolidated 32-bit Docker Build + Install + set -ex + $SCRIPT_RUN_PREFIX sh -c " + set -ex + cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0 . + make install + cp -a tests /pybind11-tests + mkdir /build-tests && cd /build-tests + cmake ../pybind11-tests ${CMAKE_EXTRA_ARGS} -DPYBIND11_WERROR=ON + make pytest -j 2" + set +ex cache: directories: - $HOME/.local/bin @@ -97,6 +153,7 @@ before_install: - | # Configure build variables + set -ex if [ "$TRAVIS_OS_NAME" = "linux" ]; then if [ -n "$CLANG" ]; then export CXX=clang++-$CLANG CC=clang-$CLANG @@ -107,18 +164,16 @@ fi export CXX=g++-$GCC CC=gcc-$GCC fi - if [ "$GCC" = "6" ]; then DOCKER=${ARCH:+$ARCH/}debian:stretch - elif [ "$GCC" = "7" ]; then DOCKER=debian:buster EXTRA_PACKAGES+=" catch python3-distutils" DOWNLOAD_CATCH=OFF - fi elif [ "$TRAVIS_OS_NAME" = "osx" ]; then export CXX=clang++ CC=clang; fi if [ -n "$CPP" ]; then CPP=-std=c++$CPP; fi if [ "${PYTHON:0:1}" = "3" ]; then PY=3; fi if [ -n "$DEBUG" ]; then CMAKE_EXTRA_ARGS+=" -DCMAKE_BUILD_TYPE=Debug"; fi + set +ex - | # Initialize environment - set -e + set -ex if [ -n "$DOCKER" ]; then docker pull $DOCKER @@ -138,7 +193,7 @@ PY_CMD=python$PYTHON if [ "$TRAVIS_OS_NAME" = "osx" ]; then if [ "$PY" = "3" ]; then - brew install python$PY; + brew update && brew upgrade python else curl -fsSL https://bootstrap.pypa.io/get-pip.py | $PY_CMD - --user fi @@ -147,13 +202,15 @@ if [ "$PY" = 3 ] || [ -n "$PYPY" ]; then $PY_CMD -m ensurepip --user fi + $PY_CMD --version $PY_CMD -m pip install --user --upgrade pip wheel fi - set +e + set +ex install: - | # Install dependencies - set -e + set -ex + cmake --version if [ -n "$DOCKER" ]; then if [ -n "$DEBUG" ]; then PY_DEBUG="python$PYTHON-dbg python$PY-scipy-dbg" @@ -165,48 +222,59 @@ libeigen3-dev libboost-dev cmake make ${EXTRA_PACKAGES} && break; done" else - if [ "$CLANG" = "5.0" ]; then - if ! [ -d ~/.local/include/c++/v1 ]; then - # Neither debian nor llvm provide a libc++ 5.0 deb; luckily it's fairly quick - # to build, install (and cache), so do it ourselves: - git clone --depth=1 https://github.com/llvm-mirror/llvm.git llvm-source - git clone https://github.com/llvm-mirror/libcxx.git llvm-source/projects/libcxx -b release_50 - git clone https://github.com/llvm-mirror/libcxxabi.git llvm-source/projects/libcxxabi -b release_50 - mkdir llvm-build && cd llvm-build - # Building llvm requires a newer cmake than is provided by the trusty container: - CMAKE_VER=cmake-3.8.0-Linux-x86_64 - curl https://cmake.org/files/v3.8/$CMAKE_VER.tar.gz | tar xz - ./$CMAKE_VER/bin/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=~/.local ../llvm-source - make -j2 install-cxxabi install-cxx - cp -a include/c++/v1/*cxxabi*.h ~/.local/include/c++/v1 - cd .. - fi - export CXXFLAGS="-isystem $HOME/.local/include/c++/v1 -stdlib=libc++" - export LDFLAGS="-L$HOME/.local/lib -fuse-ld=lld-$CLANG" - export LD_LIBRARY_PATH="$HOME/.local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - if [ "$CPP" = "-std=c++17" ]; then CPP="-std=c++1z"; fi + if [ "$CLANG" = "7" ]; then + export CXXFLAGS="-stdlib=libc++" fi export NPY_NUM_BUILD_JOBS=2 echo "Installing pytest, numpy, scipy..." - ${PYPY:+travis_wait 30} $PY_CMD -m pip install --user --upgrade pytest numpy scipy \ - ${PYPY:+--extra-index-url https://imaginary.ca/trusty-pypi} + local PIP_CMD="" + if [ -n $PYPY ]; then + # For expediency, install only versions that are available on the extra index. + travis_wait 30 \ + $PY_CMD -m pip install --user --upgrade --extra-index-url https://imaginary.ca/trusty-pypi \ + pytest numpy==1.15.4 scipy==1.2.0 + else + $PY_CMD -m pip install --user --upgrade pytest numpy scipy + fi echo "done." - wget -q -O eigen.tar.gz https://bitbucket.org/eigen/eigen/get/3.3.3.tar.gz - tar xzf eigen.tar.gz - export CMAKE_INCLUDE_PATH="${CMAKE_INCLUDE_PATH:+$CMAKE_INCLUDE_PATH:}$PWD/eigen-eigen-67e894c6cd8f" + mkdir eigen + curl -fsSL https://bitbucket.org/eigen/eigen/get/3.3.4.tar.bz2 | \ + tar --extract -j --directory=eigen --strip-components=1 + export CMAKE_INCLUDE_PATH="${CMAKE_INCLUDE_PATH:+$CMAKE_INCLUDE_PATH:}$PWD/eigen" fi - set +e + set +ex script: -- $SCRIPT_RUN_PREFIX cmake ${CMAKE_EXTRA_ARGS} - -DPYBIND11_PYTHON_VERSION=$PYTHON - -DPYBIND11_CPP_STANDARD=$CPP - -DPYBIND11_WERROR=${WERROR:-ON} - -DDOWNLOAD_CATCH=${DOWNLOAD_CATCH:-ON} -- $SCRIPT_RUN_PREFIX make pytest -j 2 -- $SCRIPT_RUN_PREFIX make cpptest -j 2 -- if [ -n "$CMAKE" ]; then $SCRIPT_RUN_PREFIX make test_cmake_build; fi +- | + # CMake Configuration + set -ex + $SCRIPT_RUN_PREFIX cmake ${CMAKE_EXTRA_ARGS} \ + -DPYBIND11_PYTHON_VERSION=$PYTHON \ + -DPYBIND11_CPP_STANDARD=$CPP \ + -DPYBIND11_WERROR=${WERROR:-ON} \ + -DDOWNLOAD_CATCH=${DOWNLOAD_CATCH:-ON} \ + . + set +ex +- | + # pytest + set -ex + $SCRIPT_RUN_PREFIX make pytest -j 2 VERBOSE=1 + set +ex +- | + # cpptest + set -ex + $SCRIPT_RUN_PREFIX make cpptest -j 2 + set +ex +- | + # CMake Build Interface + set -ex + if [ -n "$CMAKE" ]; then $SCRIPT_RUN_PREFIX make test_cmake_build; fi + set +ex after_failure: cat tests/test_cmake_build/*.log* after_script: -- if [ -n "$DOCKER" ]; then docker stop "$containerid"; docker rm "$containerid"; fi +- | + # Cleanup (Docker) + set -ex + if [ -n "$DOCKER" ]; then docker stop "$containerid"; docker rm "$containerid"; fi + set +ex diff -Nru spead2-1.10.0/aclocal.m4 spead2-2.1.0/aclocal.m4 --- spead2-1.10.0/aclocal.m4 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/aclocal.m4 2020-02-14 08:26:57.000000000 +0000 @@ -765,70 +765,6 @@ AC_DEFUN([_AM_IF_OPTION], [m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) -# Copyright (C) 1999-2017 Free Software Foundation, Inc. -# -# This file is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# _AM_PROG_CC_C_O -# --------------- -# Like AC_PROG_CC_C_O, but changed for automake. We rewrite AC_PROG_CC -# to automatically call this. -AC_DEFUN([_AM_PROG_CC_C_O], -[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl -AC_REQUIRE_AUX_FILE([compile])dnl -AC_LANG_PUSH([C])dnl -AC_CACHE_CHECK( - [whether $CC understands -c and -o together], - [am_cv_prog_cc_c_o], - [AC_LANG_CONFTEST([AC_LANG_PROGRAM([])]) - # Make sure it works both with $CC and with simple cc. - # Following AC_PROG_CC_C_O, we do the test twice because some - # compilers refuse to overwrite an existing .o file with -o, - # though they will create one. - am_cv_prog_cc_c_o=yes - for am_i in 1 2; do - if AM_RUN_LOG([$CC -c conftest.$ac_ext -o conftest2.$ac_objext]) \ - && test -f conftest2.$ac_objext; then - : OK - else - am_cv_prog_cc_c_o=no - break - fi - done - rm -f core conftest* - unset am_i]) -if test "$am_cv_prog_cc_c_o" != yes; then - # Losing compiler, so override with the script. - # FIXME: It is wrong to rewrite CC. - # But if we don't then we get into trouble of one sort or another. - # A longer-term fix would be to have automake use am__CC in this case, - # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" - CC="$am_aux_dir/compile $CC" -fi -AC_LANG_POP([C])]) - -# For backward compatibility. -AC_DEFUN_ONCE([AM_PROG_CC_C_O], [AC_REQUIRE([AC_PROG_CC])]) - -# Copyright (C) 2001-2017 Free Software Foundation, Inc. -# -# This file is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# AM_RUN_LOG(COMMAND) -# ------------------- -# Run COMMAND, save the exit status in ac_status, and log it. -# (This has been adapted from Autoconf's _AC_RUN_LOG macro.) -AC_DEFUN([AM_RUN_LOG], -[{ echo "$as_me:$LINENO: $1" >&AS_MESSAGE_LOG_FD - ($1) >&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD - (exit $ac_status); }]) - # Check to make sure that the build environment is sane. -*- Autoconf -*- # Copyright (C) 1996-2017 Free Software Foundation, Inc. diff -Nru spead2-1.10.0/bootstrap.sh spead2-2.1.0/bootstrap.sh --- spead2-1.10.0/bootstrap.sh 2018-10-11 11:59:14.000000000 +0000 +++ spead2-2.1.0/bootstrap.sh 2019-09-17 20:08:17.000000000 +0000 @@ -1,31 +1,6 @@ #!/bin/bash set -e -if [ "x$1" != "x--no-python" ]; then - if ! which trollius2asyncio >/dev/null; then - echo "trollius2asyncio not found. Install trollius-fixers or specify --no-python." 1>&2 - exit 1 - fi - - for i in send/ recv/ test/test_send_ test/test_passthrough_ tools/recv_ tools/send_ tools/bench_; do - src="spead2/${i}trollius.py" - trg="spead2/${i}asyncio.py" - echo "# Generated by bootstrap.sh from $src. Do not edit." > "$trg" - echo >> "$trg" - cat "$src" >> "$trg" - trollius2asyncio -w -n --no-diffs "$trg" - # trollius2asyncio doesn't rewrite internally references e.g - # spead2.send.trollius to spead2.send.asyncio. This is a bit hacky but - # easier than providing custom fixers and trying to bootstrap them. - # Also we can't use -i because GNU sed and OS X sed have mutually - # incompatible interpretations. - sed 's/trollius/asyncio/g' "$trg" > "$trg.tmp" - mv "$trg.tmp" "$trg" - cat >> "$trg" < include/spead2/common_ibv_loader.h +python gen/gen_ibv_loader.py cxx > src/common_ibv_loader.cpp autoreconf --install --force diff -Nru spead2-1.10.0/configure spead2-2.1.0/configure --- spead2-1.10.0/configure 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/configure 2020-02-14 08:26:58.000000000 +0000 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for spead2 1.9.2. +# Generated by GNU Autoconf 2.69 for spead2 2.0.2. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -576,8 +576,8 @@ # Identity of this package. PACKAGE_NAME='spead2' PACKAGE_TARNAME='spead2' -PACKAGE_VERSION='1.9.2' -PACKAGE_STRING='spead2 1.9.2' +PACKAGE_VERSION='2.0.2' +PACKAGE_STRING='spead2 2.0.2' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -606,15 +606,10 @@ SPEAD2_USE_MOVNTDQ SPEAD2_USE_PTHREAD_SETAFFINITY_NP SPEAD2_USE_EVENTFD +SPEAD2_USE_SENDMMSG SPEAD2_USE_RECVMMSG -SPEAD2_USE_NETMAP -am__fastdepCC_FALSE -am__fastdepCC_TRUE -CCDEPMODE -ac_ct_CC -CFLAGS -CC SPEAD2_USE_PCAP +SPEAD2_USE_IBV_MPRQ SPEAD2_USE_IBV_EXP SPEAD2_USE_IBV RANLIB @@ -709,9 +704,10 @@ enable_dependency_tracking with_ibv with_ibv_exp +with_ibv_mprq with_pcap -with_netmap with_recvmmsg +with_sendmmsg with_eventfd with_pthread_setaffinity_np with_movntdq @@ -732,9 +728,7 @@ CPPFLAGS CCC AR -RANLIB -CC -CFLAGS' +RANLIB' # Initialize some variables set by options. @@ -1285,7 +1279,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures spead2 1.9.2 to adapt to many kinds of systems. +\`configure' configures spead2 2.0.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1352,7 +1346,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of spead2 1.9.2:";; + short | recursive ) echo "Configuration of spead2 2.0.2:";; esac cat <<\_ACEOF @@ -1378,9 +1372,11 @@ --without-ibv Do not use libibverbs and librdmacm, even if detected --without-ibv-exp Do not use verbs_exp.h, even if detected + --without-ibv-mprq Do not use multi-packet receive queues, even if + detected --without-pcap Do not use pcap, even if detected - --without-netmap Do not use netmap, even if detected --without-recvmmsg Do not use recvmmsg system call + --without-sendmmsg Do not use sendmmsg system call --without-eventfd Do not use eventfd system call for semaphores --without-pthread_setaffinity_np Do not set thread affinity @@ -1399,8 +1395,6 @@ you have headers in a nonstandard directory AR Static library archiver command RANLIB Generate index to static library - CC C compiler command - CFLAGS C compiler flags Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -1468,7 +1462,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -spead2 configure 1.9.2 +spead2 configure 2.0.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1566,90 +1560,6 @@ } # ac_fn_cxx_try_link -# ac_fn_c_try_compile LINENO -# -------------------------- -# Try to compile conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_compile () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - rm -f conftest.$ac_objext - if { { ac_try="$ac_compile" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_compile") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { - test -z "$ac_c_werror_flag" || - test ! -s conftest.err - } && test -s conftest.$ac_objext; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_compile - -# ac_fn_c_try_link LINENO -# ----------------------- -# Try to link conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_link () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - rm -f conftest.$ac_objext conftest$ac_exeext - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { - test -z "$ac_c_werror_flag" || - test ! -s conftest.err - } && test -s conftest$ac_exeext && { - test "$cross_compiling" = yes || - test -x conftest$ac_exeext - }; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information - # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would - # interfere with the next link command; also delete a directory that is - # left behind by Apple's compiler. We do this before executing the actions. - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_link - # ac_fn_cxx_check_func LINENO FUNC VAR # ------------------------------------ # Tests whether FUNC exists, setting the cache variable VAR accordingly @@ -1720,7 +1630,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by spead2 $as_me 1.9.2, which was +It was created by spead2 $as_me 2.0.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2587,7 +2497,7 @@ # Define the identity of the package. PACKAGE='spead2' - VERSION='1.9.2' + VERSION='2.0.2' cat >>confdefs.h <<_ACEOF @@ -3941,6 +3851,84 @@ + SPEAD2_USE_IBV_MPRQ=0 + +# Check whether --with-ibv_mprq was given. +if test "${with_ibv_mprq+set}" = set; then : + withval=$with_ibv_mprq; +else + with_ibv_mprq=check +fi + + if test "x$with_ibv_mprq" = xno; then : + +else + SPEAD2_USE_IBV_MPRQ=1 + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for multi-packet receive queue" >&5 +$as_echo_n "checking for multi-packet receive queue... " >&6; } +if ${spead2_cv_ibv_mprq+:} false; then : + $as_echo_n "(cached) " >&6 +else + + spead2_save_LIBS=$LIBS + LIBS="-libverbs $LIBS" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include + +int +main () +{ +ibv_exp_cq_family_v1 *cq_intf; + ibv_exp_wq_family *wq_intf; + ibv_exp_qp_init_attr attr; + ibv_exp_query_intf(NULL, NULL, NULL); + ibv_exp_create_wq(NULL, NULL); + ibv_exp_create_rwq_ind_table(NULL, NULL); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + spead2_cv_ibv_mprq=yes +else + spead2_cv_ibv_mprq=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS=$spead2_save_LIBS + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $spead2_cv_ibv_mprq" >&5 +$as_echo "$spead2_cv_ibv_mprq" >&6; } + if test "x$spead2_cv_ibv_mprq" = xyes; then : + +else + SPEAD2_USE_IBV_MPRQ=0 +fi + + + + + if test "x$with_ibv_mprq" = xcheck; then : + +else + if test "x$SPEAD2_USE_IBV_MPRQ" = x0; then : + as_fn_error $? "--with-ibv_mprq was specified but not found" "$LINENO" 5 +fi +fi + +fi + + + + + + + SPEAD2_USE_PCAP=0 # Check whether --with-pcap was given. @@ -4013,863 +4001,85 @@ -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. -set dummy ${ac_tool_prefix}gcc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}gcc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } + + SPEAD2_USE_RECVMMSG=0 + +# Check whether --with-recvmmsg was given. +if test "${with_recvmmsg+set}" = set; then : + withval=$with_recvmmsg; else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } + with_recvmmsg=check fi + if test "x$with_recvmmsg" = xno; then : -fi -if test -z "$ac_cv_prog_CC"; then - ac_ct_CC=$CC - # Extract the first word of "gcc", so it can be a program name with args. -set dummy gcc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_CC+:} false; then : - $as_echo_n "(cached) " >&6 else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="gcc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + ac_fn_cxx_check_func "$LINENO" "recvmmsg" "ac_cv_func_recvmmsg" +if test "x$ac_cv_func_recvmmsg" = xyes; then : + SPEAD2_USE_RECVMMSG=1 +fi + if test "x$with_recvmmsg" = xcheck; then : + +else + if test "x$SPEAD2_USE_RECVMMSG" = x0; then : + as_fn_error $? "--with-recvmmsg was specified but not found" "$LINENO" 5 fi fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -$as_echo "$ac_ct_CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } + fi - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - CC=$ac_ct_CC - fi + + + + + + + SPEAD2_USE_SENDMMSG=0 + +# Check whether --with-sendmmsg was given. +if test "${with_sendmmsg+set}" = set; then : + withval=$with_sendmmsg; else - CC="$ac_cv_prog_CC" + with_sendmmsg=check fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. -set dummy ${ac_tool_prefix}cc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. + if test "x$with_sendmmsg" = xno; then : + else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}cc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + ac_fn_cxx_check_func "$LINENO" "sendmmsg" "ac_cv_func_sendmmsg" +if test "x$ac_cv_func_sendmmsg" = xyes; then : + SPEAD2_USE_SENDMMSG=1 +fi + + if test "x$with_sendmmsg" = xcheck; then : +else + if test "x$SPEAD2_USE_SENDMMSG" = x0; then : + as_fn_error $? "--with-sendmmsg was specified but not found" "$LINENO" 5 fi fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } + fi - fi + + + + + + SPEAD2_USE_EVENTFD=0 + +# Check whether --with-eventfd was given. +if test "${with_eventfd+set}" = set; then : + withval=$with_eventfd; +else + with_eventfd=check fi -if test -z "$CC"; then - # Extract the first word of "cc", so it can be a program name with args. -set dummy cc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else - ac_prog_rejected=no -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then - ac_prog_rejected=yes - continue - fi - ac_cv_prog_CC="cc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -if test $ac_prog_rejected = yes; then - # We found a bogon in the path, so make sure we never use it. - set dummy $ac_cv_prog_CC - shift - if test $# != 0; then - # We chose a different compiler from the bogus one. - # However, it has the same basename, so the bogon will be chosen - # first if we set CC to just the basename; use the full file name. - shift - ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" - fi -fi -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - for ac_prog in cl.exe - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$CC" && break - done -fi -if test -z "$CC"; then - ac_ct_CC=$CC - for ac_prog in cl.exe -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -$as_echo "$ac_ct_CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_CC" && break -done - - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - CC=$ac_ct_CC - fi -fi - -fi - - -test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5; } - -# Provide some information about the compiler. -$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 -set X $ac_compile -ac_compiler=$2 -for ac_option in --version -v -V -qversion; do - { { ac_try="$ac_compiler $ac_option >&5" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_compiler $ac_option >&5") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - sed '10a\ -... rest of stderr output deleted ... - 10q' conftest.err >conftest.er1 - cat conftest.er1 >&5 - fi - rm -f conftest.er1 conftest.err - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } -done - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 -$as_echo_n "checking whether we are using the GNU C compiler... " >&6; } -if ${ac_cv_c_compiler_gnu+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ -#ifndef __GNUC__ - choke me -#endif - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_compiler_gnu=yes -else - ac_compiler_gnu=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -ac_cv_c_compiler_gnu=$ac_compiler_gnu - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 -$as_echo "$ac_cv_c_compiler_gnu" >&6; } -if test $ac_compiler_gnu = yes; then - GCC=yes -else - GCC= -fi -ac_test_CFLAGS=${CFLAGS+set} -ac_save_CFLAGS=$CFLAGS -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 -$as_echo_n "checking whether $CC accepts -g... " >&6; } -if ${ac_cv_prog_cc_g+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_save_c_werror_flag=$ac_c_werror_flag - ac_c_werror_flag=yes - ac_cv_prog_cc_g=no - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_g=yes -else - CFLAGS="" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - -else - ac_c_werror_flag=$ac_save_c_werror_flag - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_g=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - ac_c_werror_flag=$ac_save_c_werror_flag -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 -$as_echo "$ac_cv_prog_cc_g" >&6; } -if test "$ac_test_CFLAGS" = set; then - CFLAGS=$ac_save_CFLAGS -elif test $ac_cv_prog_cc_g = yes; then - if test "$GCC" = yes; then - CFLAGS="-g -O2" - else - CFLAGS="-g" - fi -else - if test "$GCC" = yes; then - CFLAGS="-O2" - else - CFLAGS= - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 -$as_echo_n "checking for $CC option to accept ISO C89... " >&6; } -if ${ac_cv_prog_cc_c89+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_cv_prog_cc_c89=no -ac_save_CC=$CC -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#include -struct stat; -/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ -struct buf { int x; }; -FILE * (*rcsopen) (struct buf *, struct stat *, int); -static char *e (p, i) - char **p; - int i; -{ - return p[i]; -} -static char *f (char * (*g) (char **, int), char **p, ...) -{ - char *s; - va_list v; - va_start (v,p); - s = g (p, va_arg (v,int)); - va_end (v); - return s; -} - -/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has - function prototypes and stuff, but not '\xHH' hex character constants. - These don't provoke an error unfortunately, instead are silently treated - as 'x'. The following induces an error, until -std is added to get - proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an - array size at least. It's necessary to write '\x00'==0 to get something - that's true only with -std. */ -int osf4_cc_array ['\x00' == 0 ? 1 : -1]; - -/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters - inside strings and character constants. */ -#define FOO(x) 'x' -int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; - -int test (int i, double x); -struct s1 {int (*f) (int a);}; -struct s2 {int (*f) (double a);}; -int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); -int argc; -char **argv; -int -main () -{ -return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; - ; - return 0; -} -_ACEOF -for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ - -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" -do - CC="$ac_save_CC $ac_arg" - if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_c89=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext - test "x$ac_cv_prog_cc_c89" != "xno" && break -done -rm -f conftest.$ac_ext -CC=$ac_save_CC - -fi -# AC_CACHE_VAL -case "x$ac_cv_prog_cc_c89" in - x) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -$as_echo "none needed" >&6; } ;; - xno) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -$as_echo "unsupported" >&6; } ;; - *) - CC="$CC $ac_cv_prog_cc_c89" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 -$as_echo "$ac_cv_prog_cc_c89" >&6; } ;; -esac -if test "x$ac_cv_prog_cc_c89" != xno; then : - -fi - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC understands -c and -o together" >&5 -$as_echo_n "checking whether $CC understands -c and -o together... " >&6; } -if ${am_cv_prog_cc_c_o+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF - # Make sure it works both with $CC and with simple cc. - # Following AC_PROG_CC_C_O, we do the test twice because some - # compilers refuse to overwrite an existing .o file with -o, - # though they will create one. - am_cv_prog_cc_c_o=yes - for am_i in 1 2; do - if { echo "$as_me:$LINENO: $CC -c conftest.$ac_ext -o conftest2.$ac_objext" >&5 - ($CC -c conftest.$ac_ext -o conftest2.$ac_objext) >&5 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } \ - && test -f conftest2.$ac_objext; then - : OK - else - am_cv_prog_cc_c_o=no - break - fi - done - rm -f core conftest* - unset am_i -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_prog_cc_c_o" >&5 -$as_echo "$am_cv_prog_cc_c_o" >&6; } -if test "$am_cv_prog_cc_c_o" != yes; then - # Losing compiler, so override with the script. - # FIXME: It is wrong to rewrite CC. - # But if we don't then we get into trouble of one sort or another. - # A longer-term fix would be to have automake use am__CC in this case, - # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" - CC="$am_aux_dir/compile $CC" -fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - -depcc="$CC" am_compiler_list= - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5 -$as_echo_n "checking dependency style of $depcc... " >&6; } -if ${am_cv_CC_dependencies_compiler_type+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then - # We make a subdir and do the tests there. Otherwise we can end up - # making bogus files that we don't know about and never remove. For - # instance it was reported that on HP-UX the gcc test will end up - # making a dummy file named 'D' -- because '-MD' means "put the output - # in D". - rm -rf conftest.dir - mkdir conftest.dir - # Copy depcomp to subdir because otherwise we won't find it if we're - # using a relative directory. - cp "$am_depcomp" conftest.dir - cd conftest.dir - # We will build objects and dependencies in a subdirectory because - # it helps to detect inapplicable dependency modes. For instance - # both Tru64's cc and ICC support -MD to output dependencies as a - # side effect of compilation, but ICC will put the dependencies in - # the current directory while Tru64 will put them in the object - # directory. - mkdir sub - - am_cv_CC_dependencies_compiler_type=none - if test "$am_compiler_list" = ""; then - am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp` - fi - am__universal=false - case " $depcc " in #( - *\ -arch\ *\ -arch\ *) am__universal=true ;; - esac - - for depmode in $am_compiler_list; do - # Setup a source with many dependencies, because some compilers - # like to wrap large dependency lists on column 80 (with \), and - # we should not choose a depcomp mode which is confused by this. - # - # We need to recreate these files for each test, as the compiler may - # overwrite some of them when testing with obscure command lines. - # This happens at least with the AIX C compiler. - : > sub/conftest.c - for i in 1 2 3 4 5 6; do - echo '#include "conftst'$i'.h"' >> sub/conftest.c - # Using ": > sub/conftst$i.h" creates only sub/conftst1.h with - # Solaris 10 /bin/sh. - echo '/* dummy */' > sub/conftst$i.h - done - echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf - - # We check with '-c' and '-o' for the sake of the "dashmstdout" - # mode. It turns out that the SunPro C++ compiler does not properly - # handle '-M -o', and we need to detect this. Also, some Intel - # versions had trouble with output in subdirs. - am__obj=sub/conftest.${OBJEXT-o} - am__minus_obj="-o $am__obj" - case $depmode in - gcc) - # This depmode causes a compiler race in universal mode. - test "$am__universal" = false || continue - ;; - nosideeffect) - # After this tag, mechanisms are not by side-effect, so they'll - # only be used when explicitly requested. - if test "x$enable_dependency_tracking" = xyes; then - continue - else - break - fi - ;; - msvc7 | msvc7msys | msvisualcpp | msvcmsys) - # This compiler won't grok '-c -o', but also, the minuso test has - # not run yet. These depmodes are late enough in the game, and - # so weak that their functioning should not be impacted. - am__obj=conftest.${OBJEXT-o} - am__minus_obj= - ;; - none) break ;; - esac - if depmode=$depmode \ - source=sub/conftest.c object=$am__obj \ - depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \ - $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \ - >/dev/null 2>conftest.err && - grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 && - grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 && - grep $am__obj sub/conftest.Po > /dev/null 2>&1 && - ${MAKE-make} -s -f confmf > /dev/null 2>&1; then - # icc doesn't choke on unknown options, it will just issue warnings - # or remarks (even with -Werror). So we grep stderr for any message - # that says an option was ignored or not supported. - # When given -MP, icc 7.0 and 7.1 complain thusly: - # icc: Command line warning: ignoring option '-M'; no argument required - # The diagnosis changed in icc 8.0: - # icc: Command line remark: option '-MP' not supported - if (grep 'ignoring option' conftest.err || - grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else - am_cv_CC_dependencies_compiler_type=$depmode - break - fi - fi - done - - cd .. - rm -rf conftest.dir -else - am_cv_CC_dependencies_compiler_type=none -fi - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CC_dependencies_compiler_type" >&5 -$as_echo "$am_cv_CC_dependencies_compiler_type" >&6; } -CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type - - if - test "x$enable_dependency_tracking" != xno \ - && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then - am__fastdepCC_TRUE= - am__fastdepCC_FALSE='#' -else - am__fastdepCC_TRUE='#' - am__fastdepCC_FALSE= -fi - - - - - - SPEAD2_USE_NETMAP=0 - -# Check whether --with-netmap was given. -if test "${with_netmap+set}" = set; then : - withval=$with_netmap; -else - with_netmap=check -fi - - if test "x$with_netmap" = xno; then : - -else - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - # netmap_user.h doesn't do extern "C" - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for netmap" >&5 -$as_echo_n "checking for netmap... " >&6; } -if ${spead2_cv_header_net_netmap_user_h+:} false; then : - $as_echo_n "(cached) " >&6 -else - - spead2_save_LIBS=$LIBS - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#define NETMAP_WITH_LIBS - - #include -#include - -int -main () -{ -nm_open(NULL, NULL, 0, NULL) - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - spead2_cv_header_net_netmap_user_h=yes -else - spead2_cv_header_net_netmap_user_h=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LIBS=$spead2_save_LIBS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $spead2_cv_header_net_netmap_user_h" >&5 -$as_echo "$spead2_cv_header_net_netmap_user_h" >&6; } - if test "x$spead2_cv_header_net_netmap_user_h" = xyes; then : - SPEAD2_USE_NETMAP=1 -fi - - - ac_ext=cpp -ac_cpp='$CXXCPP $CPPFLAGS' -ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_cxx_compiler_gnu - - - - if test "x$with_netmap" = xcheck; then : - -else - if test "x$SPEAD2_USE_NETMAP" = x0; then : - as_fn_error $? "--with-netmap was specified but not found" "$LINENO" 5 -fi -fi - -fi - - - - - - - - SPEAD2_USE_RECVMMSG=0 - -# Check whether --with-recvmmsg was given. -if test "${with_recvmmsg+set}" = set; then : - withval=$with_recvmmsg; -else - with_recvmmsg=check -fi - - if test "x$with_recvmmsg" = xno; then : - -else - ac_fn_cxx_check_func "$LINENO" "recvmmsg" "ac_cv_func_recvmmsg" -if test "x$ac_cv_func_recvmmsg" = xyes; then : - SPEAD2_USE_RECVMMSG=1 -fi - - if test "x$with_recvmmsg" = xcheck; then : - -else - if test "x$SPEAD2_USE_RECVMMSG" = x0; then : - as_fn_error $? "--with-recvmmsg was specified but not found" "$LINENO" 5 -fi -fi - -fi - - - - - - - - SPEAD2_USE_EVENTFD=0 - -# Check whether --with-eventfd was given. -if test "${with_eventfd+set}" = set; then : - withval=$with_eventfd; -else - with_eventfd=check -fi - - if test "x$with_eventfd" = xno; then : - + + if test "x$with_eventfd" = xno; then : + else ac_fn_cxx_check_func "$LINENO" "eventfd" "ac_cv_func_eventfd" if test "x$ac_cv_func_eventfd" = xyes; then : @@ -5106,9 +4316,9 @@ ### Determine libraries to link against -LIBS="-lboost_system -lpthread" +LIBS="-lboost_system -lpthread -ldl" if test "x$SPEAD2_USE_IBV" = "x1"; then : - LIBS="-lrdmacm -libverbs $LIBS" + LIBS="$LIBS" fi if test "x$SPEAD2_USE_PCAP" = "x1"; then : LIBS="-lpcap $LIBS" @@ -5382,10 +4592,6 @@ as_fn_error $? "conditional \"am__fastdepCXX\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then - as_fn_error $? "conditional \"am__fastdepCC\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${DEBUG_SYMBOLS_TRUE}" && test -z "${DEBUG_SYMBOLS_FALSE}"; then as_fn_error $? "conditional \"DEBUG_SYMBOLS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -5815,7 +5021,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by spead2 $as_me 1.9.2, which was +This file was extended by spead2 $as_me 2.0.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5872,7 +5078,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -spead2 config.status 1.9.2 +spead2 config.status 2.0.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -Nru spead2-1.10.0/configure.ac spead2-2.1.0/configure.ac --- spead2-1.10.0/configure.ac 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/configure.ac 2019-10-08 08:00:52.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2016, 2017 SKA South Africa +# Copyright 2016, 2017, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -73,6 +73,23 @@ ) SPEAD2_ARG_WITH( + [ibv_mprq], + [AS_HELP_STRING([--without-ibv-mprq], [Do not use multi-packet receive queues, even if detected])], + [SPEAD2_USE_IBV_MPRQ], + [SPEAD2_USE_IBV_MPRQ=1 + SPEAD2_CHECK_FEATURE( + [ibv_mprq], [multi-packet receive queue], [infiniband/verbs_exp.h], [ibverbs], + [ibv_exp_cq_family_v1 *cq_intf; + ibv_exp_wq_family *wq_intf; + ibv_exp_qp_init_attr attr; + ibv_exp_query_intf(NULL, NULL, NULL); + ibv_exp_create_wq(NULL, NULL); + ibv_exp_create_rwq_ind_table(NULL, NULL);], + [], [SPEAD2_USE_IBV_MPRQ=0]) + ] +) + +SPEAD2_ARG_WITH( [pcap], [AS_HELP_STRING([--without-pcap], [Do not use pcap, even if detected])], [SPEAD2_USE_PCAP], @@ -85,26 +102,18 @@ SPEAD2_ARG_WITH( - [netmap], - [AS_HELP_STRING([--without-netmap], [Do not use netmap, even if detected])], - [SPEAD2_USE_NETMAP], - [AC_LANG_PUSH(C) # netmap_user.h doesn't do extern "C" - SPEAD2_CHECK_FEATURE( - [header_net_netmap_user_h], [netmap], - [stddef.h net/netmap_user.h], [], [nm_open(NULL, NULL, 0, NULL)], - [SPEAD2_USE_NETMAP=1], [], [#define NETMAP_WITH_LIBS] - ) - AC_LANG_POP - ] -) - -SPEAD2_ARG_WITH( [recvmmsg], [AS_HELP_STRING([--without-recvmmsg], [Do not use recvmmsg system call])], [SPEAD2_USE_RECVMMSG], [AC_CHECK_FUNC([recvmmsg], [SPEAD2_USE_RECVMMSG=1], [])]) SPEAD2_ARG_WITH( + [sendmmsg], + [AS_HELP_STRING([--without-sendmmsg], [Do not use sendmmsg system call])], + [SPEAD2_USE_SENDMMSG], + [AC_CHECK_FUNC([sendmmsg], [SPEAD2_USE_SENDMMSG=1], [])]) + +SPEAD2_ARG_WITH( [eventfd], [AS_HELP_STRING([--without-eventfd], [Do not use eventfd system call for semaphores])], [SPEAD2_USE_EVENTFD], @@ -155,8 +164,8 @@ ### Determine libraries to link against -LIBS="-lboost_system -lpthread" -AS_IF([test "x$SPEAD2_USE_IBV" = "x1"], [LIBS="-lrdmacm -libverbs $LIBS"]) +LIBS="-lboost_system -lpthread -ldl" +AS_IF([test "x$SPEAD2_USE_IBV" = "x1"], [LIBS="$LIBS"]) AS_IF([test "x$SPEAD2_USE_PCAP" = "x1"], [LIBS="-lpcap $LIBS"]) ### Build variants diff -Nru spead2-1.10.0/debian/changelog spead2-2.1.0/debian/changelog --- spead2-1.10.0/debian/changelog 2019-01-08 15:25:07.000000000 +0000 +++ spead2-2.1.0/debian/changelog 2020-06-08 11:40:46.000000000 +0000 @@ -1,3 +1,10 @@ +spead2 (2.1.0-1kern1) bionic; urgency=medium + + [ Athanaseus Javas Ramaila] + * New upstream version 2.1.0 + + -- KERN packaging Mon, 08 Jun 2020 13:40:46 +0200 + spead2 (1.10.0-1) bionic; urgency=medium [ Gijs Molenaar ] diff -Nru spead2-1.10.0/doc/changelog.rst spead2-2.1.0/doc/changelog.rst --- spead2-1.10.0/doc/changelog.rst 2018-12-11 11:29:22.000000000 +0000 +++ spead2-2.1.0/doc/changelog.rst 2020-02-14 08:27:06.000000000 +0000 @@ -1,6 +1,140 @@ Changelog ========= +.. rubric:: 2.1.0 + +- Support unicast receive with ibverbs acceleration (including in + :program:`mcdump`). +- Fix :program:`spead2_recv` listening only on loopback when given just a port + number. +- Support unicast addresses in a few APIs that previously only accepted + multicast addresses; in most cases the unicast address must match the + interface address. +- Add missing ```` include to ````. +- Show the values of immediate items in :program:`spead2_recv`. +- Fix occasional crash when using thread pool with more than one thread + together with ibverbs. +- Fix bug in mcdump causing it to hang if the arguments couldn't be parsed + (only happened when capturing to file). +- Fix :program:`spead2_recv` reporting statistics that may miss out the last + batch of packets. + +.. rubric:: 2.0.2 + +- Log warnings on some internal errors (that hopefully never happen). +- Include wheels for Python 3.8. +- Build debug symbols for binary wheels (in a separate tarball on Github). + +.. rubric:: 2.0.1 + +- Fix race condition in TCP receiver (#78). +- Update vendored pybind11 to 2.4.2. + +.. rubric:: 2.0.0 + +- Drop support for Python 2. +- Drop support for Python 3.4. +- Drop support for trollius. +- Drop support for netmap. +- Avoid creating some cyclic references. These were not memory leaks, but + prevented CPython from freeing objects as soon as it might have. +- Update vendored pybind11 to 2.4.1. + +.. rubric:: 1.14.0 + +- Add `new_order` argument to :py:meth:`spead2.ItemGroup.update`. +- Improved unit tests. + +.. rubric:: 1.13.1 + +- Raise :exc:`ValueError` on a dtype that has zero itemsize (#37). +- Change exception when dtype has embedded objects from :exc:`TypeError` to + :exc:`ValueError` for consistency +- Remove duplicated socket handle in UDP receiver (#67). +- Make `max_poll` argument to :py:class:`spead2.send.UdpIbvStream` actually + have an effect (#55). +- Correctly report EOF errors in :cpp:class:`spead2::send::streambuf_stream`. +- Wrap implicitly computed heap cnts to the number of available bits (#3). + Previously behaviour was undefined. +- Some header files were not installed by ``make install`` (#72). + +.. rubric:: 1.13.0 + +- Significant performance improvements to send code (in some cases an order of + magnitude improvement). +- Add :option:`--max-heap` option to :program:`spead2_send` and + :program:`spead2_send.py` to control the depth of the send queue. +- Change the meaning of the :option:`--heaps` option in :program:`spead2_bench` + and :program:`spead2_bench.py`: it now also controls the depth of the sending + queue. +- Fix a bug in send rate limiting that could allow the target rate to be + exceeded under some conditions. +- Remove :option:`--threads` option from C++ :program:`spead2_send`, as the new + optimised implementation isn't thread-safe. +- Disable the ``test_numpy_large`` test on macOS, which was causing frequent + failures on TravisCI due to dropped packets. + +.. rubric:: 1.12.0 + +- Provide manylinux2010 wheels. +- Dynamically link to libibverbs and librdmacm on demand. This allows binaries + (particularly wheels) to support verbs acceleration but still work on systems + without these libraries installed. +- Support for Boost 1.70. Unfortunately Boost 1.70 removes the ability to query + the io_service from a socket, so constructors that take a socket but no + io_service are omitted when compiling with Boost 1.70 or newer. +- Fix some compiler warnings from GCC 8. + +.. rubric:: 1.11.4 + +- Rework the locking internals of :cpp:class:`spead2::recv::stream` so that + a full ringbuffer doesn't block new readers from being added. This changes + the interfaces between :cpp:class:`spead2::recv::reader` and + :cpp:class:`spead2::recv::stream_base`, but since users generally don't deal + with that interface the major version hasn't been incremented. +- Fix a spurious log message if an in-process receiver is manually stopped. +- Fix an intermittent unit test failure due to timing. + +.. rubric:: 1.11.3 + +- Undo the optimisation of using a single flow steering rule to cover multiple + multicast groups (see #11). + +.. rubric:: 1.11.2 + +- Fix ``-c`` option to :program:`mcdump`. +- Fix a missing ``#include`` that could be exposed by including headers in a + particular order. +- Make :cpp:class:`spead2::recv::heap`'s move constructor and move assignment + operator ``noexcept``. +- Add a `long_description` to the Python metadata. + +.. rubric:: 1.11.1 + +- Update type stubs for new features in 1.11.0. + +.. rubric:: 1.11.0 + +- Add :py:attr:`spead2.recv.Stream.allow_unsized_heaps` to support rejecting + packets without a heap length. +- Add extended custom memcpy support (C++ only) for scattering data from + packets. + +.. rubric:: 1.10.1 + +- Use ibverbs multi-packet receive queues automatically when available + (supported by mlx5 driver). +- Automatically reduce buffer size for verbs receiver to match hardware limits + (fixed #64). +- Gracefully handle Ctrl-C in :program:`spead2_recv` and print statistics. +- Add typing stub files to assist checking with Mypy. +- Give a name to the argument of + :py:meth:`spead2.recv.Stream.add_inproc_reader`. +- Fix Python binding for one of the UDP reader overloads that takes an existing + socket. This was a deprecated overload. +- Add a unit test for ibverbs support. It's not run by default because it + needs specific hardware. + .. rubric:: 1.10.0 - Accelerate per-packet processing, particularly when `max_heaps` is large. diff -Nru spead2-1.10.0/doc/cpp-netmap.rst spead2-2.1.0/doc/cpp-netmap.rst --- spead2-1.10.0/doc/cpp-netmap.rst 2018-12-11 11:04:22.000000000 +0000 +++ spead2-2.1.0/doc/cpp-netmap.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ -Support for netmap -================== - -.. warning:: - - Support for netmap is **deprecated** and will be removed in a future release - of spead2. It is not being maintained or tested at present, and the - :doc:`verbs ` support is much easier to use. - -Introduction ------------- -As an experimental feature, it is possible to use the netmap_ framework to -receive packets at a higher rate than is possible with the regular sockets -API. This is particularly useful for small packets. - -.. _netmap: info.iet.unipi.it/~luigi/netmap/ - -This is not for the faint of heart: it requires root access, it can easily -hang the whole machine, and it imposes limitations, including: - -- Only the C++ API is supported. If you need every drop of performance, you - shouldn't be using Python anyway. -- Only Linux is currently tested. It should be theoretically possible to - support FreeBSD, but you're on your own (patches welcome). -- Only IPv4 is supported. -- Fragmented IP packets, and IP headers with optional fields are not - supported. -- Checksums are not validated (although possibly the NIC will check them). -- Only one reader is supported per network interface. -- All packets that arrive with the correct UDP port will be processed, - regardless of destination address. This could mean, for example, that - unrelated multicast streams will be processed even though they aren't - wanted. - -Usage ------ -Once netmap is installed and the header file :file:`net/netmap_user.h` is placed in -a system include directory, pass :makevar:`NETMAP=1` to :program:`make` to include netmap -support in the library. - -Then, instead of :cpp:class:`spead2::recv::udp_reader`, use -:cpp:class:`spead2::recv::netmap_udp_reader`. - -.. doxygenclass:: spead2::recv::netmap_udp_reader - :members: netmap_udp_reader diff -Nru spead2-1.10.0/doc/cpp-recv.rst spead2-2.1.0/doc/cpp-recv.rst --- spead2-1.10.0/doc/cpp-recv.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/cpp-recv.rst 2019-03-06 09:53:45.000000000 +0000 @@ -35,8 +35,11 @@ :cpp:class:`spead2::recv::stream` and implement :cpp:func:`heap_ready` and optionally override :cpp:func:`stop_received`. +Note that some public functions are incorrectly listed as protected below due +to limitations of the documentation tools. + .. doxygenclass:: spead2::recv::stream - :members: emplace_reader, stop, stop_received, flush, heap_ready, get_stats + :members: .. doxygenstruct:: spead2::recv::stream_stats :members: @@ -79,8 +82,61 @@ For an allocator set on a stream, a pointer to a :cpp:class:`spead2::recv::packet_header` is passed as a hint to the allocator, allowing memory to be placed according to information in the packet. Note that -this can be any packet from the heap, so you must not rely on it being the -initial packet. +for unreliable transport this could be any packet from the heap, and you should +not rely on it being the initial packet. .. doxygenclass:: spead2::memory_allocator :members: allocate, free + + +Custom memory scatter +--------------------------- +In specialised high-bandwidth cases, the overhead of assembling heaps in +temporary storage before scattering the data into other arrangements can be +very high. It is possible (since 1.11) to take complete control over the +transfer of the payload of the SPEAD packets. Before embarking on such an +approach, be sure you have a good understanding of the SPEAD protocol, +particularly packets, heaps, item pointers and payload. + +In the simplest case, each heap needs to be written to some special or +pre-allocated storage, but in a contiguous fashion. In this case it is +sufficient to provide a custom allocator (see above), which will return a +pointer to the target storage. + +In more complex cases, the contents of each heap, or even each packet, needs +to be scattered to discontiguous storage areas. In this case, one can +additionally override the memory copy function with +:cpp:func:`~spead2::recv::stream_base::set_memcpy` and providing a +:cpp:type:`~spead2::recv::packet_memcpy_function`. + +.. doxygentypedef:: spead2::recv::packet_memcpy_function + +It takes a pointer to the start of the heap's allocation (as returned by the +allocator) and the packet metadata. The default implementation is equivalent +to the following: + +.. code-block:: c++ + + void copy(const spead2::memory_allocator::pointer &allocation, const packet_header &packet) + { + memcpy(allocation.get() + packet.payload_offset, packet.payload, packet.payload_length); + } + +Note that when providing your own memory copy and allocator, you don't +necessarily need the allocator to actually return a pointer to payload memory. +It could, for example, populate a structure that guides the copy, and return a +pointer to that; or it could return a null pointer. There are some caveats +though: + +1. If the sender doesn't provide the heap length item, then spead2 may need to + make multiple allocations of increasing size as the heap grows, and each + time it will copy (with standard memcpy, rather than your custom one) the + old content to the new. Assuming you aren't expecting such packets, you can + reject them using + :cpp:func:`~spead2::recv::stream_base::set_allow_unsized_heaps`. + +2. :cpp:func:`spead2::recv::heap_base::get_items` constructs pointers to the items + on the assumption of the default memcpy function, so if your replacement + doesn't copy things to the same place, you obviously won't be able to use + those pointers. Note that :cpp:func:`~spead2::recv::heap::get_descriptors` + will also not be usable. diff -Nru spead2-1.10.0/doc/cpp.rst spead2-2.1.0/doc/cpp.rst --- spead2-1.10.0/doc/cpp.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/cpp.rst 2019-10-08 08:00:52.000000000 +0000 @@ -39,4 +39,3 @@ cpp-inproc cpp-logging cpp-ibverbs - cpp-netmap diff -Nru spead2-1.10.0/doc/Doxyfile spead2-2.1.0/doc/Doxyfile --- spead2-1.10.0/doc/Doxyfile 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/Doxyfile 2019-10-08 08:00:52.000000000 +0000 @@ -50,13 +50,13 @@ SORT_MEMBERS_CTORS_1ST = YES SORT_BY_SCOPE_NAME = YES MAX_INITIALIZER_LINES = 2 -PREDEFINED = SPEAD2_USE_NETMAP=1 SPEAD2_USE_IBV=1 SPEAD2_USE_PCAP=1 +PREDEFINED = SPEAD2_USE_IBV=1 SPEAD2_USE_PCAP=1 QUIET = YES WARN_IF_UNDOCUMENTED = NO INPUT = ../src ../include/spead2 -EXCLUDE_PATTERNS = test_* +EXCLUDE_PATTERNS = test_* spead2_* GENERATE_HTML = NO GENERATE_LATEX = NO diff -Nru spead2-1.10.0/doc/introduction.rst spead2-2.1.0/doc/introduction.rst --- spead2-1.10.0/doc/introduction.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/introduction.rst 2019-10-08 08:00:52.000000000 +0000 @@ -12,26 +12,25 @@ - cleanly supports several SPEAD flavours (e.g. 64-40 and 64-48) in one module, with the receiver adapting to the flavour used by the sender; - supports Python 3; -- supports asynchronous operation, using trollius_ or asyncio_. +- supports asynchronous operation, using asyncio_. .. _SPEAD: https://casper.berkeley.edu/wiki/SPEAD .. _PySPEAD: https://github.com/ska-sa/PySPEAD/ -.. _trollius: http://trollius.readthedocs.io/ .. _asyncio: https://docs.python.org/3/library/asyncio.html Preparation ----------- -There is optional support for netmap_ and ibverbs_ for higher performance, and +There is optional support for ibverbs_ for higher performance, and pcap_ for reading from previously captured packet dumps. If the libraries (including development headers) are installed, they will be detected automatically and support for them will be included. -.. _netmap: https://github.com/luigirizzo/netmap .. _ibverbs: https://www.openfabrics.org/downloads/libibverbs/README.html .. _pcap: http://www.tcpdump.org/ If you are installing spead2 from a git checkout, it is first necessary to run -``./bootstrap.sh`` to prepare the configure script and related files. When +``./bootstrap.sh`` to prepare the configure script and related files. This +requires a Python installation with pycparser and jinja2 installed. When building from a packaged download this is not required. High-performance usage requires larger buffer sizes than Linux allows by @@ -46,55 +45,46 @@ Installing spead2 for Python ---------------------------- -The only Python dependencies are numpy_ and six_, and trollius_ on Python -versions below 3.4 (for 3.4-3.6 trollius can still be used, and is needed to -run the test suite). Running the test -suite additionally requires nose_, decorator_ and netifaces_, and some tests -depend on PySPEAD_ (they will be skipped if it is not installed). It is also -necessary to have the development headers for Python. +The only Python dependency is numpy_. + +The test suite has additional dependencies; refer to +:file:`setup.py` if you are developing spead2. There are two ways to install spead2 for Python: compiling from source and -installing a binary wheel. The binary wheels are experimental and only -recommended if installing from source is not an option. +installing a binary wheel. .. _numpy: http://www.numpy.org .. _six: https://pythonhosted.org/six/ -.. _nose: https://nose.readthedocs.io/en/latest/ -.. _decorator: http://pythonhosted.org/decorator/ -.. _netifaces: https://pypi.python.org/pypi/netifaces -Python install from source -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Installing from source requires a modern C++ compiler supporting C++11 (GCC -4.8+ or Clang 3.5+) as well as Boost (including compiled libraries). At the -moment only GNU/Linux and OS X get tested but other POSIX-like systems should -work too. There are no plans to support Windows. +Installing a binary wheel +^^^^^^^^^^^^^^^^^^^^^^^^^ +As from version 1.12, binary wheels are provided on PyPI for x86-64 Linux +systems. These support all the optional features, and it is now the recommended +installation method as it does not depend on a compiler, development +libraries etc. The wheels use the "manylinux2010" tag, which requires at least +:command:`pip` 19.0 to install. -Installation works with standard Python installation methods. For example, to -install the latest version from PyPI, run:: +Provided your system meets these requirements, just run:: pip install spead2 -Installing a binary wheel -^^^^^^^^^^^^^^^^^^^^^^^^^ -As from version 1.3.2, binary wheels for x86-64 Linux systems are placed on the -Github `release page`_. They are still experimental, lack the optional features, -and may be slower than installs from source because they are compiled with an -old compiler. They are mainly intended for systems where it is not practical -to install a new enough C++ compiler or Boost. For this reason, they are -currently *not* provided through PyPI. - -.. _release page: https://github.com/ska-sa/spead2/releases +Python install from source +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Installing from source requires a modern C++ compiler supporting C++11 (GCC +4.8+ or Clang 3.5+, although only GCC 5.4 and Clang 3.8 are tested and support +for older compilers may be dropped) as well as Boost (including compiled +libraries) and the Python development headers. At the moment only GNU/Linux and +OS X get tested but other POSIX-like systems should work too. There are no +plans to support Windows. -After downloading the appropriate wheel for your Python version, install it -with :samp:`pip install {filename}`. +Installation works with standard Python installation methods. Installing spead2 for C++ ------------------------- -spead2 requires a modern C++ compiler supporting C++11 (GCC 4.8+ or Clang 3.5+) -as well as Boost (including compiled libraries). At the moment only GNU/Linux -and OS X get tested but other POSIX-like systems should work too. There are no -plans to support Windows. +spead2 requires a modern C++ compiler supporting C++11 (see above for supported +compilers) as well as Boost (including compiled libraries). At the moment only +GNU/Linux and OS X get tested but other POSIX-like systems should work too. +There are no plans to support Windows. The C++ API uses the standard autoconf installation flow i.e.: diff -Nru spead2-1.10.0/doc/perf.rst spead2-2.1.0/doc/perf.rst --- spead2-1.10.0/doc/perf.rst 2018-12-11 11:05:49.000000000 +0000 +++ spead2-2.1.0/doc/perf.rst 2019-10-08 08:00:52.000000000 +0000 @@ -167,17 +167,16 @@ Kernel bypass APIs ^^^^^^^^^^^^^^^^^^ -There are two low-level kernel bypass networking APIs supported: -:doc:`ibverbs ` and :doc:`netmap ` (although the latter -is deprecated and will be removed from a future spead2 release). These provide -a zero-copy path from the NIC into the spead2 library, without the kernel being -involved. This can make a huge performance difference, particularly for small -packet sizes. +Kernel bypass APIs provide a zero-copy path from the NIC into the spead2 +library, without the kernel being involved. This can make a huge performance +difference, particularly for small packet sizes. These APIs are not a free lunch: they will only work with some NICs, require special kernel drivers and setup, have limitations in what networking features they can support, and require the application to specify which network device -to use. Refer to the links above for more details. +to use. + +Currently only one kernel bypass API is supported: `ibverbs `. Memory allocation ^^^^^^^^^^^^^^^^^ diff -Nru spead2-1.10.0/doc/py-ibverbs.rst spead2-2.1.0/doc/py-ibverbs.rst --- spead2-1.10.0/doc/py-ibverbs.rst 2018-12-11 11:08:17.000000000 +0000 +++ spead2-2.1.0/doc/py-ibverbs.rst 2020-02-12 09:25:32.000000000 +0000 @@ -8,9 +8,9 @@ There are a number of limitations in the current implementation: - - Only IPv4 is supported - - VLAN tagging, IP optional headers, and IP fragmentation are not supported - - Only multicast is supported. + - Only IPv4 is supported. + - VLAN tagging, IP optional headers, and IP fragmentation are not supported. + - For sending, only multicast is supported. Within these limitations, it is quite easy to take advantage of this faster code path. The main difficulty is that one *must* specify the IP address of @@ -55,14 +55,17 @@ place of `endpoints`. :param list endpoints: List of 2-tuples, each containing a - hostname/IP address the multicast group and the UDP port number. + hostname/IP address and the UDP port number. The address may be either + unicast or multicast, but in the former case there must be + an interface with that IP address (usually it will be the same as + `interface_address`, but this is not required as an interface may have + multiple IP addresses). :param str interface_address: Hostname/IP address of the interface which will be subscribed :param int max_size: Maximum packet size that will be accepted - :param int buffer_size: Requested memory allocation for work requests. Note - that this is used to determine the number of packets - to buffer; if the packets are smaller than `max_size`, - then fewer bytes will be buffered. + :param int buffer_size: Requested memory allocation for work requests. + It may be adjusted due to implementation-dependent limits or alignment + requirements. :param int comp_vector: Completion channel vector (interrupt) for asynchronous operation, or a negative value to poll continuously. Polling @@ -78,8 +81,18 @@ non-negative) or letting other code run on the thread (if `comp_vector` is negative). -Reducing `max_size` to be close to the actual packet size can make a -significant performance improvement. +If supported by the NIC, the receive code will automatically use a +"multi-packet receive queue", which allows each packet to consume only the +amount of space needed in the buffer. This is not supported by all NICs (for +example, ConnectX-3 does not, but ConnectX-5 does). When in use, the +`max_size` parameter has little impact on performance, and is used only to +reject larger packets. + +When multi-packet receive queues are not supported, performance can be +improved by making `max_size` as small as possible for the intended data +stream. This will increase the number of packets that can be buffered (because +the buffer is divided into fixed-size slots), and also improve memory +efficiency by keeping data more-or-less contiguous. Environment variables ^^^^^^^^^^^^^^^^^^^^^ @@ -99,10 +112,8 @@ ------- Sending is done by using the class :py:class:`spead2.send.UdpIbvStream` instead of :py:class:`spead2.send.UdpStream`. It has a different constructor, but the -same methods. There are also :py:class:`spead2.send.asyncio.UdpIbvStream` and -:py:class:`spead2.send.trollius.UdpIbvStream` classes, analogous to -:py:class:`spead2.send.asyncio.UdpStream` and -:py:class:`spead2.send.trollius.UdpStream`. +same methods. There is also a :py:class:`spead2.send.asyncio.UdpIbvStream` +class, analogous to :py:class:`spead2.send.asyncio.UdpStream`. .. py:class:: spead2.send.UdpIbvStream(thread_pool, multicast_group, port, config, interface_address, buffer_size, ttl=1, comp_vector=0, max_poll=DEFAULT_MAX_POLL) diff -Nru spead2-1.10.0/doc/py-protocol.rst spead2-2.1.0/doc/py-protocol.rst --- spead2-1.10.0/doc/py-protocol.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/py-protocol.rst 2019-10-07 14:23:49.000000000 +0000 @@ -3,9 +3,8 @@ * Any descriptor with a numpy header is handled by numpy. The value is converted to native endian, but is otherwise left untouched. * Strings are expected to use ASCII encoding only. At present this is variably - enforced, and enforcement may differ between Python 2 and 3. Future versions - may apply stricter enforcement. This applies to names, descriptions, and to - values passed with the `c` format code. + enforced. Future versions may apply stricter enforcement. This applies to + names, descriptions, and to values passed with the `c` format code. * The `c` format code may only be used with length 8, and `f` may only be used with lengths 32 or 64. * The `0` format code is not supported. diff -Nru spead2-1.10.0/doc/py-recv.rst spead2-2.1.0/doc/py-recv.rst --- spead2-1.10.0/doc/py-recv.rst 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/doc/py-recv.rst 2020-02-12 09:25:32.000000000 +0000 @@ -112,7 +112,9 @@ .. py:method:: add_udp_reader(multicast_group, port, max_size=DEFAULT_UDP_MAX_SIZE, buffer_size=DEFAULT_UDP_BUFFER_SIZE, interface_address) - Feed data from a UDP port with multicast (IPv4 only). + Feed data from a UDP port (IPv4 only). This is intended for use with + multicast, but it will also accept a unicast address as long as it is the + same as the interface address. :param str multicast_group: Hostname/IP address of the multicast group to subscribe to :param int port: UDP port number @@ -213,14 +215,18 @@ treatment. Such heaps can then be detected with :meth:`~spead2.recv.Heap.is_end_of_stream`. + .. py:attribute:: allow_unsized_heaps + + By default, spead2 caters for heaps without a `HEAP_LEN` item, and will + dynamically extend the memory allocation as data arrives. However, this + can be expensive, and ideally senders should include this item. Setting + this attribute to ``False`` will cause packets without this item to be + rejected. + Asynchronous receive ^^^^^^^^^^^^^^^^^^^^ -Asynchronous I/O is supported through Python 3's :py:mod:`asyncio` module, as -well as through trollius_ (a Python 2 backport). It can be combined with other -asynchronous I/O frameworks like twisted_ and Tornado_. - -The documentation below is for the :py:mod:`asyncio` interface; replace all -instances of ``asyncio`` with ``trollius`` if you're using trollius. +Asynchronous I/O is supported through Python's :py:mod:`asyncio` module. It can +be combined with other asynchronous I/O frameworks like twisted_ and Tornado_. .. py:class:: spead2.recv.asyncio.Stream(\*args, \*\*kwargs, loop=None) @@ -240,12 +246,11 @@ :param loop: asyncio event loop to use, overriding constructor. -.. _trollius: http://trollius.readthedocs.io/ .. _twisted: https://twistedmatrix.com/trac/ .. _tornado: http://www.tornadoweb.org/en/stable/ -When using Python 3.5 or higher, the stream is also asynchronously iterable, -i.e., can be used in an ``async for`` loop to iterate over the heaps. +The stream is also asynchronously iterable, i.e., can be used in an ``async +for`` loop to iterate over the heaps. .. _py-memory-allocators: diff -Nru spead2-1.10.0/doc/py-send.rst spead2-2.1.0/doc/py-send.rst --- spead2-1.10.0/doc/py-send.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/py-send.rst 2019-11-13 08:46:10.000000000 +0000 @@ -41,6 +41,39 @@ .. automethod:: spead2.send.HeapGenerator.get_start .. automethod:: spead2.send.HeapGenerator.get_end +.. py:class:: spead2.send.Heap(flavour=spead2.Flavour()) + + .. py:attribute:: repeat_pointers + + Enable/disable repetition of item pointers in all packets. + + Usually this is not needed, but it can enable some specialised use + cases where immediates can be recovered from incomplete heaps or where + the receiver examines the item pointers in each packet to decide how + to handle it. The packet size must be large enough to fit all the item + pointers for the heap (the implementation also reserves a little space, + so do not rely on a tight fit working). + + The default is disabled. + + .. py:method:: add_item(item) + + Add an :py:class:`~spead2.Item` to the heap. This references the memory in + the item rather than copying it. It does *not* cause a descriptor to be + sent; use :py:meth:`add_descriptor` for that. + + .. py:method:: add_descriptor(descriptor) + + Add a :py:class:`~spead2.Descriptor` to the heap. + + .. py:method:: add_start() + + Convenience method to add a start-of-stream item. + + .. py:method:: add_end() + + Convenience method to add an end-of-stream item. + Blocking send ------------- @@ -54,7 +87,7 @@ Sends a :py:class:`spead2.send.Heap` to the peer, and wait for completion. There is currently no indication of whether it successfully - arrived. + arrived, but :py:exc:`IOError` is raised if it could not be sent. If not specified, a heap cnt is chosen automatically (the choice can be modified by calling :py:meth:`set_cnt_sequence`). If a non-negative value @@ -67,11 +100,14 @@ will have cnt `next`, and each following cnt will be incremented by `step`. When using this, it is the user's responsibility to ensure that the generated values remain unique. The initial state is `next` = - 1, `cnt` = 1. + 1, `step` = 1. This is useful when multiple senders will send heaps to the same receiver, and need to keep their heap cnts separate. + If the computed cnt overflows the number of bits available, the + bottom-most bits are taken. + UDP ^^^ @@ -234,8 +270,7 @@ Asynchronous send ----------------- -As for asynchronous receives, asynchronous sends are managed by asyncio_ or -trollius_. A +As for asynchronous receives, asynchronous sends are managed by asyncio_. A stream can buffer up multiple heaps for asynchronous send, up to the limit specified by `max_heaps` in the :py:class:`~spead2.send.StreamConfig`. If this limit is exceeded, heaps will be dropped, and the returned future has an @@ -243,13 +278,11 @@ low-level error in sending the heap (for example, if the packet size exceeds the MTU). -.. _trollius: http://trollius.readthedocs.io/ .. _asyncio: https://docs.python.org/3/library/asyncio.html -The classes existing in the :py:mod:`spead2.send.asyncio` and -:py:mod:`spead2.send.trollius` modules, and mostly implement the same -constructors as the synchronous classes. They implement the following abstract -interface (the class does not actually exist): +The classes exist in the :py:mod:`spead2.send.asyncio` modules, and mostly +implement the same constructors as the synchronous classes. They implement the +following abstract interface (the class does not actually exist): .. class:: spead2.send.asyncio.AbstractStream() diff -Nru spead2-1.10.0/doc/tools.rst spead2-2.1.0/doc/tools.rst --- spead2-1.10.0/doc/tools.rst 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/doc/tools.rst 2020-02-12 09:25:32.000000000 +0000 @@ -6,7 +6,7 @@ spead2_bench ------------ A benchmarking tool is provided to estimate the maximum throughput for UDP. -There are two versions: one implemented in Python (:program:`spead2_bench.py`) +There are two versions: one implemented in Python 3 (:program:`spead2_bench.py`) and one in C++ (:program:`spead2_bench`), which are installed by the corresponding installers. The examples show the Python version, but the C++ version functions very similarly. However, they cannot be mixed: use the same @@ -33,7 +33,7 @@ spead2_send/spead2_recv ----------------------- There are also separate :program:`spead2_send` and :program:`spead2_recv` (and -Python equivalents) programs. The former generates a stream of meaningless +Python 3 equivalent) programs. The former generates a stream of meaningless data, while the latter consumes an existing stream and reports the heaps and items that it finds. Apart from being useful for debugging a stream, :program:`spead2_recv` has a similar plethora of command-line options for @@ -44,9 +44,9 @@ mcdump ------ mcdump is a tool similar to tcpdump_, but specialised for high-speed capture of -multicast UDP traffic using hardware that supports the Infiniband Verbs API. It -has only been tested on Mellanox ConnectX-3 NICs. Like gulp_, it uses a -separate thread for disk I/O and CPU core affinity to achieve reliable +UDP traffic using hardware that supports the Infiniband Verbs API. It +has only been tested on Mellanox ConnectX-3 and ConnectX-5 NICs. Like gulp_, it +uses a separate thread for disk I/O and CPU core affinity to achieve reliable performance. With a sufficiently fast disk subsystem, it is able to capture line rate from a 40Gb/s adapter. @@ -78,6 +78,11 @@ continues until interrupted by :kbd:`Ctrl-C`. You can also list more :samp:`{group}:{port}` pairs, which will all stored in the same pcap file. +While originally written for multicast, mcdump also supports unicast. An IP +address must still be provided; usually it will be the same as the interface +address, but it could be a different address if the interface has multiple IP +addresses. + You can also specify ``-`` in place of the filename to suppress the write to file. This is useful to simply count the bytes/packets received without being limited by disk throughput. @@ -115,7 +120,7 @@ detected at compile time. Otherwise, all packets have a zero timestamp in the file. -- Only IPv4 multicast is supported. +- Only IPv4 is supported. - It is not optimised for small packets (below about 1KB). Packet capture rates top out around 6Mpps for current hardware. diff -Nru spead2-1.10.0/examples/Makefile.in spead2-2.1.0/examples/Makefile.in --- spead2-1.10.0/examples/Makefile.in 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/examples/Makefile.in 2020-02-14 08:26:58.000000000 +0000 @@ -206,9 +206,6 @@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXDEPMODE = @CXXDEPMODE@ @@ -247,19 +244,19 @@ SPEAD2_USE_EVENTFD = @SPEAD2_USE_EVENTFD@ SPEAD2_USE_IBV = @SPEAD2_USE_IBV@ SPEAD2_USE_IBV_EXP = @SPEAD2_USE_IBV_EXP@ +SPEAD2_USE_IBV_MPRQ = @SPEAD2_USE_IBV_MPRQ@ SPEAD2_USE_MOVNTDQ = @SPEAD2_USE_MOVNTDQ@ -SPEAD2_USE_NETMAP = @SPEAD2_USE_NETMAP@ SPEAD2_USE_PCAP = @SPEAD2_USE_PCAP@ SPEAD2_USE_POSIX_SEMAPHORES = @SPEAD2_USE_POSIX_SEMAPHORES@ SPEAD2_USE_PTHREAD_SETAFFINITY_NP = @SPEAD2_USE_PTHREAD_SETAFFINITY_NP@ SPEAD2_USE_RECVMMSG = @SPEAD2_USE_RECVMMSG@ +SPEAD2_USE_SENDMMSG = @SPEAD2_USE_SENDMMSG@ STRIP = @STRIP@ VERSION = @VERSION@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ diff -Nru spead2-1.10.0/examples/test_recv.py spead2-2.1.0/examples/test_recv.py --- spead2-1.10.0/examples/test_recv.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/examples/test_recv.py 2018-12-13 11:13:00.000000000 +0000 @@ -18,7 +18,6 @@ from __future__ import print_function, division import spead2 import spead2.recv -import sys import logging logging.basicConfig(level=logging.INFO) diff -Nru spead2-1.10.0/examples/test_ringbuffer.cpp spead2-2.1.0/examples/test_ringbuffer.cpp --- spead2-1.10.0/examples/test_ringbuffer.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/examples/test_ringbuffer.cpp 2019-04-30 16:10:58.000000000 +0000 @@ -131,7 +131,7 @@ (void) item; } } - catch (spead2::ringbuffer_stopped) + catch (spead2::ringbuffer_stopped &) { } } diff -Nru spead2-1.10.0/examples/test_send_asyncio.py spead2-2.1.0/examples/test_send_asyncio.py --- spead2-1.10.0/examples/test_send_asyncio.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/examples/test_send_asyncio.py 2019-09-17 20:08:17.000000000 +0000 @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2015, 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import logging +import asyncio + +import numpy as np + +import spead2 +import spead2.send +import spead2.send.asyncio + + +logging.basicConfig(level=logging.INFO) + +thread_pool = spead2.ThreadPool() +stream = spead2.send.asyncio.UdpStream( + thread_pool, "127.0.0.1", 8888, spead2.send.StreamConfig(rate=1e7)) +del thread_pool # Make sure this doesn't crash anything + +shape = (40, 50) +ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, spead2.BUG_COMPAT_PYSPEAD_0_5_2)) +item = ig.add_item(0x1234, 'foo', 'a foo item', shape=shape, dtype=np.int32) +item.value = np.zeros(shape, np.int32) +futures = [ + stream.async_send_heap(ig.get_heap()), + stream.async_send_heap(ig.get_end()) +] +# Delete things to check that there are no refcounting bugs +del ig +del stream +asyncio.get_event_loop().run_until_complete(asyncio.wait(futures)) diff -Nru spead2-1.10.0/examples/test_send.py spead2-2.1.0/examples/test_send.py --- spead2-1.10.0/examples/test_send.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/examples/test_send.py 2018-12-13 11:12:55.000000000 +0000 @@ -17,7 +17,6 @@ import spead2 import spead2.send -import sys import logging import numpy as np diff -Nru spead2-1.10.0/examples/test_send_trollius.py spead2-2.1.0/examples/test_send_trollius.py --- spead2-1.10.0/examples/test_send_trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/examples/test_send_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -import spead2 -import spead2.send -import spead2.send.trollius -import sys -import logging -import numpy as np -import trollius - -logging.basicConfig(level=logging.INFO) - -thread_pool = spead2.ThreadPool() -stream = spead2.send.trollius.UdpStream( - thread_pool, "127.0.0.1", 8888, spead2.send.StreamConfig(rate=1e7)) -del thread_pool # Make sure this doesn't crash anything - -shape = (40, 50) -ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, spead2.BUG_COMPAT_PYSPEAD_0_5_2)) -item = ig.add_item(0x1234, 'foo', 'a foo item', shape=shape, dtype=np.int32) -item.value = np.zeros(shape, np.int32) -futures = [ - stream.async_send_heap(ig.get_heap()), - stream.async_send_heap(ig.get_end()) -] -# Delete things to check that there are no refcounting bugs -del ig -del stream -trollius.get_event_loop().run_until_complete(trollius.wait(futures)) diff -Nru spead2-1.10.0/.flake8 spead2-2.1.0/.flake8 --- spead2-1.10.0/.flake8 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/.flake8 2019-10-08 08:00:52.000000000 +0000 @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 100 +exclude = 3rdparty doc/conf.py venv pypy* +ignore = E203 W503 W504 diff -Nru spead2-1.10.0/gen/gen_ibv_loader.py spead2-2.1.0/gen/gen_ibv_loader.py --- spead2-1.10.0/gen/gen_ibv_loader.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/gen/gen_ibv_loader.py 2019-05-03 07:50:32.000000000 +0000 @@ -0,0 +1,321 @@ +#!/usr/bin/env python + +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import sys +import copy + +import jinja2 +from pycparser import c_ast +from pycparser.c_parser import CParser +from pycparser.c_generator import CGenerator + + +PREFIX = ''' +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * This file is automatically generated. Do not edit. + */ +''' + +# Extracted signatures for symbols needed from libibverbs. The typedefs are +# arbitrary and just used to allow pycparser to parse the code. +# +# Note that some functions in infiniband/verbs.h are implemented as static +# inline functions, and so do not get listed here. +INPUT = ''' +typedef unsigned long size_t; +typedef unsigned long uint64_t; + +void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents); + +struct ibv_pd *ibv_alloc_pd(struct ibv_context *context); + +int ibv_close_device(struct ibv_context *context); + +struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context); + +struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, + void *cq_context, + struct ibv_comp_channel *channel, + int comp_vector); + +struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr); + +int ibv_dealloc_pd(struct ibv_pd *pd); + +int ibv_dereg_mr(struct ibv_mr *mr); + +int ibv_destroy_comp_channel(struct ibv_comp_channel *channel); + +int ibv_destroy_cq(struct ibv_cq *cq); + +int ibv_destroy_qp(struct ibv_qp *qp); + +void ibv_free_device_list(struct ibv_device **list); + +int ibv_get_cq_event(struct ibv_comp_channel *channel, + struct ibv_cq **cq, void **cq_context); + +uint64_t ibv_get_device_guid(struct ibv_device *device); + +struct ibv_device **ibv_get_device_list(int *num_devices); + +struct ibv_context *ibv_open_device(struct ibv_device *device); + +int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask); + +int ibv_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr); + +struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, + size_t length, int access); + + +int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr); + +struct rdma_event_channel *rdma_create_event_channel(void); + +int rdma_create_id(struct rdma_event_channel *channel, + struct rdma_cm_id **id, void *context, + enum rdma_port_space ps); + +void rdma_destroy_event_channel(struct rdma_event_channel *channel); + +int rdma_destroy_id(struct rdma_cm_id *id); + +''' + +HEADER = PREFIX + '''\ +#ifndef SPEAD2_COMMON_IBV_LOADER_H +#define SPEAD2_COMMON_IBV_LOADER_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include + +namespace spead2 +{ + +{% for node in nodes -%} +extern {{ node | ptr | gen }}; +{% endfor %} + +// Load ibverbs. If it could not be loaded, throws std::system_error +void ibv_loader_init(); + +} // namespace spead2 + +#endif // SPEAD2_USE_IBV +#endif // SPEAD2_COMMON_IBV_LOADER_H +''' + +CXX = '''\ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include +#include +#include +#include + +namespace spead2 +{ + +static std::once_flag init_once; +static std::exception_ptr init_result; + +{% for node in nodes %} +{{ node | rename(node.name + '_stub') | gen }} +{ +{% for arg in (node | args) %} + (void) {{ arg }}; +{% endfor %} + ibv_loader_stub(init_result); +} +{% endfor %} + +{% for node in nodes %} +{{ node | ptr | gen }} = {{ node.name }}_stub; +{% endfor %} + +static void reset_stubs() +{ +{% for node in nodes %} + {{ node.name }} = {{ node.name }}_stub; +{% endfor %} +} + +static void init() +{ + try + { + dl_handle librdmacm("librdmacm.so.1"); + dl_handle libibverbs("libibverbs.so.1"); +{% for node in nodes %} +{% set lib = 'libibverbs' if node.name.startswith('ibv_') else 'librdmacm' %} + {{ node.name }} = reinterpret_cast<{{ node | ptr(False) | gen }}>( + {{ lib }}.sym("{{ node.name }}")); +{% endfor %} + // Prevent the libraries being closed, so that the symbols stay valid + librdmacm.release(); + libibverbs.release(); + } + catch (std::system_error &e) + { + init_result = std::current_exception(); + reset_stubs(); + log_warning("could not load ibverbs: %s", e.what()); + } +} + +void ibv_loader_init() +{ + std::call_once(init_once, init); + if (init_result) + std::rethrow_exception(init_result); +} + +} // namespace spead2 + +/* Wrappers in the global namespace. This is needed because ibv_exp_create_qp + * calls ibv_create_qp, and so we need to provide an implementation. + */ +{% for node in nodes if node.name in ['ibv_create_qp'] %} +{{ node | gen }} +{ + return spead2::{{ node.name }}({{ node | args | join(', ') }}); +} +{% endfor %} + +#endif // SPEAD2_USE_IBV +''' + + +class RenameVisitor(c_ast.NodeVisitor): + """Renames a function in the AST. + + The function name is stored both in the Decl and in the inner-most TypeDecl + of the return type. This only handles the latter. + """ + def __init__(self, old_name, new_name): + super(RenameVisitor, self).__init__() + self.old_name = old_name + self.new_name = new_name + + def visit_TypeDecl(self, node): + if node.declname == self.old_name: + node.declname = self.new_name + + +def rename_func(func, new_name): + """Return a copy of a function declaration with a new name""" + func = copy.deepcopy(func) + RenameVisitor(func.name, new_name).visit(func) + func.name = new_name + return func + + +def make_func_ptr(func, with_name=True): + """Create a node of pointer-to-function type.""" + if not with_name: + node = rename_func(func, None) + else: + node = copy.deepcopy(func) + node.type = c_ast.PtrDecl(quals=[], type=node.type) + return node + + +def func_args(func): + """Get list of function argument names""" + args = [arg.name for arg in func.type.args.params] + # Handle (void) + if args == [None]: + args = [] + return args + + +class Visitor(c_ast.NodeVisitor): + """Collects all the function definitions""" + def __init__(self): + self.nodes = [] + + def visit_Decl(self, node): + if not isinstance(node.type, c_ast.FuncDecl): + return + self.nodes.append(node) + + +def gen_node(node): + return CGenerator().visit(node) + + +def main(argv): + environment = jinja2.Environment(autoescape=False, trim_blocks=True) + environment.filters['gen'] = gen_node + environment.filters['rename'] = rename_func + environment.filters['ptr'] = make_func_ptr + environment.filters['args'] = func_args + header = environment.from_string(HEADER) + cxx = environment.from_string(CXX) + + ast = CParser().parse(INPUT) + visitor = Visitor() + visitor.visit(ast) + header_text = header.render(nodes=visitor.nodes) + cxx_text = cxx.render(nodes=visitor.nodes) + if len(argv) != 2 or argv[1] not in {'header', 'cxx'}: + print('Usage: {} header|cxx'.format(argv[0], file=sys.stderr)) + return 1 + elif argv[1] == 'header': + print(header_text) + else: + print(cxx_text) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff -Nru spead2-1.10.0/include/Makefile.am spead2-2.1.0/include/Makefile.am --- spead2-1.10.0/include/Makefile.am 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/Makefile.am 2019-11-13 08:46:10.000000000 +0000 @@ -20,6 +20,8 @@ spead2/common_features.h \ spead2/common_flavour.h \ spead2/common_ibv.h \ + spead2/common_ibv_loader.h \ + spead2/common_ibv_loader_utils.h \ spead2/common_inproc.h \ spead2/common_logging.h \ spead2/common_memcpy.h \ @@ -28,6 +30,7 @@ spead2/common_raw_packet.h \ spead2/common_ringbuffer.h \ spead2/common_semaphore.h \ + spead2/common_socket.h \ spead2/common_thread_pool.h \ spead2/common_unbounded_queue.h \ spead2/portable_endian.h \ @@ -35,14 +38,15 @@ spead2/recv_inproc.h \ spead2/recv_live_heap.h \ spead2/recv_mem.h \ - spead2/recv_netmap.h \ spead2/recv_packet.h \ spead2/recv_reader.h \ spead2/recv_ring_stream.h \ spead2/recv_stream.h \ + spead2/recv_tcp.h \ spead2/recv_udp_base.h \ spead2/recv_udp.h \ spead2/recv_udp_ibv.h \ + spead2/recv_udp_ibv_mprq.h \ spead2/recv_udp_pcap.h \ spead2/recv_utils.h \ spead2/send_heap.h \ diff -Nru spead2-1.10.0/include/Makefile.in spead2-2.1.0/include/Makefile.in --- spead2-1.10.0/include/Makefile.in 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/include/Makefile.in 2020-02-14 08:26:58.000000000 +0000 @@ -190,9 +190,6 @@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXDEPMODE = @CXXDEPMODE@ @@ -230,19 +227,19 @@ SPEAD2_USE_EVENTFD = @SPEAD2_USE_EVENTFD@ SPEAD2_USE_IBV = @SPEAD2_USE_IBV@ SPEAD2_USE_IBV_EXP = @SPEAD2_USE_IBV_EXP@ +SPEAD2_USE_IBV_MPRQ = @SPEAD2_USE_IBV_MPRQ@ SPEAD2_USE_MOVNTDQ = @SPEAD2_USE_MOVNTDQ@ -SPEAD2_USE_NETMAP = @SPEAD2_USE_NETMAP@ SPEAD2_USE_PCAP = @SPEAD2_USE_PCAP@ SPEAD2_USE_POSIX_SEMAPHORES = @SPEAD2_USE_POSIX_SEMAPHORES@ SPEAD2_USE_PTHREAD_SETAFFINITY_NP = @SPEAD2_USE_PTHREAD_SETAFFINITY_NP@ SPEAD2_USE_RECVMMSG = @SPEAD2_USE_RECVMMSG@ +SPEAD2_USE_SENDMMSG = @SPEAD2_USE_SENDMMSG@ STRIP = @STRIP@ VERSION = @VERSION@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ @@ -289,6 +286,8 @@ spead2/common_features.h \ spead2/common_flavour.h \ spead2/common_ibv.h \ + spead2/common_ibv_loader.h \ + spead2/common_ibv_loader_utils.h \ spead2/common_inproc.h \ spead2/common_logging.h \ spead2/common_memcpy.h \ @@ -297,6 +296,7 @@ spead2/common_raw_packet.h \ spead2/common_ringbuffer.h \ spead2/common_semaphore.h \ + spead2/common_socket.h \ spead2/common_thread_pool.h \ spead2/common_unbounded_queue.h \ spead2/portable_endian.h \ @@ -304,14 +304,15 @@ spead2/recv_inproc.h \ spead2/recv_live_heap.h \ spead2/recv_mem.h \ - spead2/recv_netmap.h \ spead2/recv_packet.h \ spead2/recv_reader.h \ spead2/recv_ring_stream.h \ spead2/recv_stream.h \ + spead2/recv_tcp.h \ spead2/recv_udp_base.h \ spead2/recv_udp.h \ spead2/recv_udp_ibv.h \ + spead2/recv_udp_ibv_mprq.h \ spead2/recv_udp_pcap.h \ spead2/recv_utils.h \ spead2/send_heap.h \ diff -Nru spead2-1.10.0/include/spead2/common_defines.h spead2-2.1.0/include/spead2/common_defines.h --- spead2-1.10.0/include/spead2/common_defines.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_defines.h 2019-03-06 09:53:45.000000000 +0000 @@ -27,6 +27,7 @@ #include #include #include +#include #ifndef SPEAD2_MAX_LOG_LEVEL #define SPEAD2_MAX_LOG_LEVEL (spead2::log_level::info) @@ -88,7 +89,7 @@ MEMCPY_NONTEMPORAL }; -typedef void *(*memcpy_function)(void * __restrict__, const void * __restrict__, std::size_t); +typedef std::function memcpy_function; /** * An unpacked descriptor. diff -Nru spead2-1.10.0/include/spead2/common_features.h.in spead2-2.1.0/include/spead2/common_features.h.in --- spead2-1.10.0/include/spead2/common_features.h.in 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_features.h.in 2019-10-08 08:00:52.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,13 +24,14 @@ #define SPEAD2_COMMON_FEATURES_H #define SPEAD2_USE_IBV @SPEAD2_USE_IBV@ -#define SPEAD2_USE_IBV_EXP @SPEAD2_USE_IBV_EXP@ +#define SPEAD2_USE_IBV_EXP (SPEAD2_USE_IBV && @SPEAD2_USE_IBV_EXP@) +#define SPEAD2_USE_IBV_MPRQ (SPEAD2_USE_IBV_EXP && @SPEAD2_USE_IBV_MPRQ@) #define SPEAD2_USE_RECVMMSG @SPEAD2_USE_RECVMMSG@ +#define SPEAD2_USE_SENDMMSG @SPEAD2_USE_SENDMMSG@ #define SPEAD2_USE_EVENTFD @SPEAD2_USE_EVENTFD@ #define SPEAD2_USE_PTHREAD_SETAFFINITY_NP @SPEAD2_USE_PTHREAD_SETAFFINITY_NP@ #define SPEAD2_USE_MOVNTDQ @SPEAD2_USE_MOVNTDQ@ #define SPEAD2_USE_POSIX_SEMAPHORES @SPEAD2_USE_POSIX_SEMAPHORES@ -#define SPEAD2_USE_NETMAP @SPEAD2_USE_NETMAP@ #define SPEAD2_USE_PCAP @SPEAD2_USE_PCAP@ #endif // SPEAD2_COMMON_FEATURES_H diff -Nru spead2-1.10.0/include/spead2/common_ibv.h spead2-2.1.0/include/spead2/common_ibv.h --- spead2-1.10.0/include/spead2/common_ibv.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_ibv.h 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016, 2017 SKA South Africa +/* Copyright 2016-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -25,10 +25,14 @@ # define _GNU_SOURCE #endif #include +#include #include +#include +#include #include #include #include +#include #if SPEAD2_USE_IBV @@ -60,6 +64,14 @@ } }; +struct ibv_context_deleter +{ + void operator()(ibv_context *ctx) + { + ibv_close_device(ctx); + } +}; + struct ibv_qp_deleter { void operator()(ibv_qp *qp) @@ -108,6 +120,46 @@ } }; +#if SPEAD2_USE_IBV_MPRQ + +struct ibv_exp_wq_deleter +{ + void operator()(ibv_exp_wq *wq) + { + ibv_exp_destroy_wq(wq); + } +}; + +struct ibv_exp_rwq_ind_table_deleter +{ + void operator()(ibv_exp_rwq_ind_table *table) + { + ibv_exp_destroy_rwq_ind_table(table); + } +}; + +class ibv_intf_deleter +{ +private: + struct ibv_context *context; + +public: + explicit ibv_intf_deleter(struct ibv_context *context = nullptr) noexcept; + void operator()(void *intf); +}; + +class ibv_exp_res_domain_deleter +{ +private: + struct ibv_context *context; + +public: + explicit ibv_exp_res_domain_deleter(struct ibv_context *context = nullptr) noexcept; + void operator()(ibv_exp_res_domain *res_domain); +}; + +#endif + } // namespace detail class rdma_event_channel_t : public std::unique_ptr @@ -123,6 +175,24 @@ rdma_cm_id_t(const rdma_event_channel_t &cm_id, void *context, rdma_port_space ps); void bind_addr(const boost::asio::ip::address &addr); + ibv_device_attr query_device() const; +#if SPEAD2_USE_IBV_EXP + ibv_exp_device_attr exp_query_device() const; +#endif +}; + +/* This class is not intended to be used for anything. However, the mlx5 driver + * will only enable multicast loopback if there at least 2 device contexts, and + * multiple instances of rdma_cm_id_t bound to the same device end up with the + * same device context, so constructing one is a way to force multicast + * loopback to function. + */ +class ibv_context_t : public std::unique_ptr +{ +public: + ibv_context_t() = default; + explicit ibv_context_t(struct ibv_device *device); + explicit ibv_context_t(const boost::asio::ip::address &addr); }; class ibv_comp_channel_t : public std::unique_ptr @@ -133,7 +203,8 @@ /// Create a file descriptor that is ready to read when the completion channel has events boost::asio::posix::stream_descriptor wrap(boost::asio::io_service &io_service) const; - void get_event(ibv_cq **cq, void **context); + /// Get an event, if one is available + bool get_event(ibv_cq **cq, void **context); }; class ibv_cq_t : public std::unique_ptr @@ -172,6 +243,9 @@ public: ibv_qp_t() = default; ibv_qp_t(const ibv_pd_t &pd, ibv_qp_init_attr *init_attr); +#if SPEAD2_USE_IBV_MPRQ + ibv_qp_t(const rdma_cm_id_t &cm_id, ibv_exp_qp_init_attr *init_attr); +#endif void modify(ibv_qp_attr *attr, int attr_mask); void modify(ibv_qp_state qp_state); @@ -196,6 +270,83 @@ ibv_flow_t(const ibv_qp_t &qp, ibv_flow_attr *flow); }; +/** + * Create a single flow rule. + * + * @pre The endpoint contains an IPv4 address. + */ +ibv_flow_t create_flow( + const ibv_qp_t &qp, const boost::asio::ip::udp::endpoint &endpoint, + int port_num); + +/** + * Create flow rules to subscribe to a given set of endpoints. + * + * If the address in an endpoint is unspecified, it will not be filtered on. + * Multicast addresses are supported; unicast addresses must have corresponding + * interfaces (which are used to retrieve the corresponding MAC addresses). + * + * @pre The @a endpoints are IPv4 addresses. + */ +std::vector create_flows( + const ibv_qp_t &qp, const std::vector &endpoints, + int port_num); + +#if SPEAD2_USE_IBV_MPRQ + +class ibv_exp_query_intf_error_category : public std::error_category +{ +public: + virtual const char *name() const noexcept override; + virtual std::string message(int condition) const override; + virtual std::error_condition default_error_condition(int condition) const noexcept override; +}; + +std::error_category &ibv_exp_query_intf_category(); + +class ibv_exp_cq_family_v1_t : public std::unique_ptr +{ +public: + ibv_exp_cq_family_v1_t() = default; + ibv_exp_cq_family_v1_t(const rdma_cm_id_t &cm_id, const ibv_cq_t &cq); +}; + +class ibv_exp_wq_t : public std::unique_ptr +{ +public: + ibv_exp_wq_t() = default; + ibv_exp_wq_t(const rdma_cm_id_t &cm_id, ibv_exp_wq_init_attr *attr); + + void modify(ibv_exp_wq_state state); +}; + +class ibv_exp_wq_family_t : public std::unique_ptr +{ +public: + ibv_exp_wq_family_t() = default; + ibv_exp_wq_family_t(const rdma_cm_id_t &cm_id, const ibv_exp_wq_t &wq); +}; + +class ibv_exp_rwq_ind_table_t : public std::unique_ptr +{ +public: + ibv_exp_rwq_ind_table_t() = default; + ibv_exp_rwq_ind_table_t(const rdma_cm_id_t &cm_id, ibv_exp_rwq_ind_table_init_attr *attr); +}; + +/// Construct a table with a single entry +ibv_exp_rwq_ind_table_t create_rwq_ind_table( + const rdma_cm_id_t &cm_id, const ibv_pd_t &pd, const ibv_exp_wq_t &wq); + +class ibv_exp_res_domain_t : public std::unique_ptr +{ +public: + ibv_exp_res_domain_t() = default; + ibv_exp_res_domain_t(const rdma_cm_id_t &cm_id, ibv_exp_res_domain_init_attr *attr); +}; + +#endif // SPEAD2_USE_IBV_MPRQ + } // namespace spead2 #endif // SPEAD2_USE_IBV diff -Nru spead2-1.10.0/include/spead2/common_ibv_loader.h spead2-2.1.0/include/spead2/common_ibv_loader.h --- spead2-1.10.0/include/spead2/common_ibv_loader.h 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_ibv_loader.h 2020-02-14 08:26:56.000000000 +0000 @@ -0,0 +1,70 @@ + +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * This file is automatically generated. Do not edit. + */ +#ifndef SPEAD2_COMMON_IBV_LOADER_H +#define SPEAD2_COMMON_IBV_LOADER_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include + +namespace spead2 +{ + +extern void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents); +extern struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context); +extern int (*ibv_close_device)(struct ibv_context *context); +extern struct ibv_comp_channel *(*ibv_create_comp_channel)(struct ibv_context *context); +extern struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector); +extern struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr); +extern int (*ibv_dealloc_pd)(struct ibv_pd *pd); +extern int (*ibv_dereg_mr)(struct ibv_mr *mr); +extern int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel); +extern int (*ibv_destroy_cq)(struct ibv_cq *cq); +extern int (*ibv_destroy_qp)(struct ibv_qp *qp); +extern void (*ibv_free_device_list)(struct ibv_device **list); +extern int (*ibv_get_cq_event)(struct ibv_comp_channel *channel, struct ibv_cq **cq, void **cq_context); +extern uint64_t (*ibv_get_device_guid)(struct ibv_device *device); +extern struct ibv_device **(*ibv_get_device_list)(int *num_devices); +extern struct ibv_context *(*ibv_open_device)(struct ibv_device *device); +extern int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); +extern int (*ibv_query_device)(struct ibv_context *context, struct ibv_device_attr *device_attr); +extern struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr, size_t length, int access); +extern int (*rdma_bind_addr)(struct rdma_cm_id *id, struct sockaddr *addr); +extern struct rdma_event_channel *(*rdma_create_event_channel)(void); +extern int (*rdma_create_id)(struct rdma_event_channel *channel, struct rdma_cm_id **id, void *context, enum rdma_port_space ps); +extern void (*rdma_destroy_event_channel)(struct rdma_event_channel *channel); +extern int (*rdma_destroy_id)(struct rdma_cm_id *id); + +// Load ibverbs. If it could not be loaded, throws std::system_error +void ibv_loader_init(); + +} // namespace spead2 + +#endif // SPEAD2_USE_IBV +#endif // SPEAD2_COMMON_IBV_LOADER_H diff -Nru spead2-1.10.0/include/spead2/common_ibv_loader_utils.h spead2-2.1.0/include/spead2/common_ibv_loader_utils.h --- spead2-1.10.0/include/spead2/common_ibv_loader_utils.h 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_ibv_loader_utils.h 2019-05-03 07:50:32.000000000 +0000 @@ -0,0 +1,72 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + */ + +#ifndef SPEAD2_COMMON_IBV_LOADER_UTILS_H +#define SPEAD2_COMMON_IBV_LOADER_UTILS_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include +#include + +namespace spead2 +{ + +enum class ibv_loader_error : int +{ + LIBRARY_ERROR, + SYMBOL_ERROR, + NO_INIT +}; + +class ibv_loader_error_category : public std::error_category +{ +public: + virtual const char *name() const noexcept override; + virtual std::string message(int condition) const override; + virtual std::error_condition default_error_condition(int condition) const noexcept override; +}; + +std::error_category &ibv_loader_category(); + +[[noreturn]] void ibv_loader_stub(std::exception_ptr init_result); + +class dl_handle +{ +private: + void *handle; +public: + dl_handle(const char *filename); + ~dl_handle(); + + void *sym(const char *name); + void *release(); +}; + +} // namespace spead2 + +#endif // SPEAD2_USE_IBV +#endif // SPEAD2_COMMON_IBV_LOADER_UTILS_H diff -Nru spead2-1.10.0/include/spead2/common_logging.h spead2-2.1.0/include/spead2/common_logging.h --- spead2-1.10.0/include/spead2/common_logging.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_logging.h 2019-11-01 13:01:18.000000000 +0000 @@ -27,6 +27,7 @@ #include #include #include +#include namespace spead2 { @@ -38,12 +39,14 @@ debug = 2 }; +std::ostream &operator<<(std::ostream &o, log_level level); + namespace detail { void log_msg_impl(log_level level, const std::string &msg); -static inline void apply_format(boost::format &format) +static inline void apply_format(boost::format &) { } diff -Nru spead2-1.10.0/include/spead2/common_memory_allocator.h spead2-2.1.0/include/spead2/common_memory_allocator.h --- spead2-1.10.0/include/spead2/common_memory_allocator.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_memory_allocator.h 2019-03-06 09:53:45.000000000 +0000 @@ -66,6 +66,12 @@ * Allocate @a size bytes of memory. The default implementation uses @c new * and pre-faults the memory. * + * The pointer type includes a custom deleter that takes a + * void * argument, which will be passed to @ref free. This + * can be used to pass extra information that is needed to free the + * memory. It is guaranteed that the base class will set this to + * @c nullptr. + * * @param size Number of bytes to allocate * @param hint Usage-dependent extra information * @returns Pointer to newly allocated memory @@ -81,7 +87,7 @@ * Free memory previously returned from @ref allocate. * * @param ptr Value returned by @ref allocate - * @param user User-defined handle returned by @ref allocate + * @param user User-defined handle stored in the deleter by @ref allocate */ virtual void free(std::uint8_t *ptr, void *user); }; diff -Nru spead2-1.10.0/include/spead2/common_socket.h spead2-2.1.0/include/spead2/common_socket.h --- spead2-1.10.0/include/spead2/common_socket.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/common_socket.h 2019-04-30 16:11:26.000000000 +0000 @@ -51,6 +51,41 @@ template void set_socket_recv_buffer_size(SocketType &socket, std::size_t buffer_size); +#if BOOST_VERSION >= 107000 +/** + * Get an object suitable for constructing another socket with the same IO + * context. The return type depends on the Boost version. This is necessary + * because Boost 1.70 removed @c basic_socket::get_io_service (as well as + * @c basic_socket::get_io_context). + */ +template +static inline typename SocketType::executor_type get_socket_io_service(SocketType &socket) +{ + return socket.get_executor(); +} +#else +template +static inline boost::asio::io_service &get_socket_io_service(SocketType &socket) +{ + return socket.get_io_service(); +} +#endif + +/** + * Determine whether a socket is using a particular IO context. This is necessary + * because Boost 1.70 removed @c basic_socket::get_io_service (as well as + * @c basic_socket::get_io_context). + */ +template +static inline bool socket_uses_io_service(SocketType &socket, boost::asio::io_service &io_service) +{ +#if BOOST_VERSION >= 107000 + return socket.get_executor() == io_service.get_executor(); +#else + return &socket.get_io_service() == &io_service; +#endif +} + } // namespace spead2 #endif // SPEAD2_COMMON_SOCKET_H diff -Nru spead2-1.10.0/include/spead2/.gitignore spead2-2.1.0/include/spead2/.gitignore --- spead2-1.10.0/include/spead2/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/include/spead2/.gitignore 2019-05-03 07:50:32.000000000 +0000 @@ -0,0 +1 @@ +common_ibv_loader.h diff -Nru spead2-1.10.0/include/spead2/py_common.h spead2-2.1.0/include/spead2/py_common.h --- spead2-1.10.0/include/spead2/py_common.h 2018-09-05 10:29:43.000000000 +0000 +++ spead2-2.1.0/include/spead2/py_common.h 2019-04-30 16:10:58.000000000 +0000 @@ -102,6 +102,10 @@ extern template class socket_wrapper; extern template class socket_wrapper; +boost::asio::ip::address make_address_no_release( + boost::asio::io_service &io_service, const std::string &hostname, + boost::asio::ip::resolver_query_base::flags flags); + /** * Issue a Python deprecation. * @@ -340,7 +344,7 @@ { fd = src.attr("fileno")().cast(); } - catch (std::exception) + catch (std::exception &) { return false; } diff -Nru spead2-1.10.0/include/spead2/recv_heap.h spead2-2.1.0/include/spead2/recv_heap.h --- spead2-1.10.0/include/spead2/recv_heap.h 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_heap.h 2019-10-28 19:03:23.000000000 +0000 @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,7 @@ /**@}*/ /* Copy inline immediate items and fix up pointers to it */ - void transfer_immediates(heap_base &&other); + void transfer_immediates(heap_base &&other) noexcept; protected: /// Create the structures from a live heap, destroying it in the process. @@ -92,8 +93,8 @@ public: heap_base() = default; - heap_base(heap_base &&other); - heap_base &operator=(heap_base &&other); + heap_base(heap_base &&other) noexcept; + heap_base &operator=(heap_base &&other) noexcept; /// Get heap ID s_item_pointer_t get_cnt() const { return cnt; } diff -Nru spead2-1.10.0/include/spead2/recv_live_heap.h spead2-2.1.0/include/spead2/recv_live_heap.h --- spead2-1.10.0/include/spead2/recv_live_heap.h 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_live_heap.h 2019-03-06 09:53:45.000000000 +0000 @@ -47,6 +47,9 @@ namespace recv { +typedef std::function packet_memcpy_function; + class heap; /** @@ -145,7 +148,6 @@ * @a exact is false, then a doubling heuristic will be used. */ void payload_reserve(std::size_t size, bool exact, const packet_header &packet, - const memcpy_function &memcpy, memory_allocator &allocator); /** @@ -181,7 +183,8 @@ * - inconsistent heap length * - payload range is beyond the heap length */ - bool add_packet(const packet_header &packet, const memcpy_function &memcpy, + bool add_packet(const packet_header &packet, + const packet_memcpy_function &packet_memcpy, memory_allocator &allocator); /// True if the heap is complete bool is_complete() const; diff -Nru spead2-1.10.0/include/spead2/recv_netmap.h spead2-2.1.0/include/spead2/recv_netmap.h --- spead2-1.10.0/include/spead2/recv_netmap.h 2018-12-11 11:18:57.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_netmap.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -/* Copyright 2015 SKA South Africa - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more - * details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -/** - * @file - * - * Support for netmap. - */ - -#ifndef SPEAD2_RECV_NETMAP_UDP_READER_H -#define SPEAD2_RECV_NETMAP_UDP_READER_H - -#include -#if SPEAD2_USE_NETMAP - -#define NETMAP_WITH_LIBS -#include -#include -#include -#include -#include -#include - -namespace spead2 -{ -namespace recv -{ - -namespace detail -{ - -class nm_desc_destructor -{ -public: - void operator()(nm_desc *) const; -}; - -} // namespace detail - -class netmap_udp_reader : public reader -{ -private: - /// File handle for the netmap mapping, usable with asio - boost::asio::posix::stream_descriptor handle; - /// Information about the netmap mapping - std::unique_ptr desc; - /// UDP port to listen on - uint16_t port; - - /// Start an asynchronous receive - void enqueue_receive(); - - /// Callback on completion of asynchronous notification - void packet_handler(const boost::system::error_code &error); - -public: - /** - * Constructor. - * - * @param owner Owning stream - * @param device Name of the network interface e.g., @c eth0 - * @param port UDP port number to listen to - */ - netmap_udp_reader(stream &owner, const std::string &device, uint16_t port); - - virtual void stop() override; -}; - -} // namespace recv -} // namespace spead2 - -#endif // SPEAD2_USE_NETMAP - -#endif // SPEAD2_RECV_NETMAP_UDP_READER_H diff -Nru spead2-1.10.0/include/spead2/recv_reader.h spead2-2.1.0/include/spead2/recv_reader.h --- spead2-1.10.0/include/spead2/recv_reader.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_reader.h 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -38,16 +38,20 @@ * a stream. Subclasses will usually override @ref stop. * * The lifecycle of a reader is: - * - construction (strand held) - * - stop (strand held) - * - join (strand not held) - * - destruction (strand held) + * - construction + * - @ref stop (called with @ref stream_base::queue_mutex held) + * - destruction + * + * All of the above occur with @ref stream::reader_mutex held. + * + * Once the reader has completed its work (whether because @ref stop was called or + * because of network input), it must call @ref stopped to indicate that + * it can be safely destroyed. */ class reader { private: stream &owner; ///< Owning stream - std::promise stopped_promise; ///< Promise filled when last completion handler done protected: /// Called by last completion handler @@ -61,19 +65,19 @@ stream &get_stream() const { return owner; } /** - * Retrieve the wrapped stream's base class. This must only be used when - * the stream's strand is held. + * Retrieve the wrapped stream's base class. This is normally only used + * to construct a @ref stream_base::add_packet_state. */ stream_base &get_stream_base() const; - /// Retrieve the io_service corresponding to the owner + /// Retrieve the @c io_service corresponding to the owner boost::asio::io_service &get_io_service(); /** * Cancel any pending asynchronous operations. This is called with the - * owner's strand held. This function does not need to wait for - * completion handlers to run, but if there are any, the destructor must - * wait for them. + * owner's queue_mutex and reader_mutex held. This function does not need + * to wait for completion handlers to run, but it must schedule a call to + * @ref stopped. */ virtual void stop() = 0; @@ -83,12 +87,6 @@ * should be given when the consumer is applying back-pressure. */ virtual bool lossy() const; - - /** - * Block until @ref stopped has been called by the last completion - * handler. This function is called outside the strand. - */ - void join(); }; /** diff -Nru spead2-1.10.0/include/spead2/recv_ring_stream.h spead2-2.1.0/include/spead2/recv_ring_stream.h --- spead2-1.10.0/include/spead2/recv_ring_stream.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_ring_stream.h 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -168,10 +168,14 @@ } catch (ringbuffer_full &e) { + bool lossy = is_lossy(); if (lossy) log_warning("worker thread blocked by full ringbuffer on heap %d", h.get_cnt()); - stats.worker_blocked++; + { + std::lock_guard lock(stats_mutex); + stats.worker_blocked++; + } ready_heaps.push(std::move(h)); if (lossy) log_debug("worker thread unblocked, heap %d pushed", h.get_cnt()); @@ -249,9 +253,9 @@ void ring_stream::stop() { /* Make sure the ringbuffer is stopped *before* the base implementation - * takes the strand. Without this, a heap_ready call could be blocking the - * strand, waiting for space in the ring buffer. This will cause the - * heap_ready call to abort, allowing the strand to be accessed for the + * takes the queue_mutex. Without this, a heap_ready call could be holding + * the mutex, waiting for space in the ring buffer. This will cause the + * heap_ready call to abort, allowing the mutex to be taken for the * rest of the shutdown. */ ready_heaps.stop(); diff -Nru spead2-1.10.0/include/spead2/recv_stream.h spead2-2.1.0/include/spead2/recv_stream.h --- spead2-1.10.0/include/spead2/recv_stream.h 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_stream.h 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017, 2018 SKA South Africa +/* Copyright 2015, 2017-2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -36,6 +36,7 @@ #include #include #include +#include namespace spead2 { @@ -126,9 +127,31 @@ * lead to all heaps in the same bucket. So rather than using * std::unordered_map, we use a custom hash table implementation (with a * fixed number of buckets). + * + * @internal + * + * Avoiding deadlocks requires a careful design with several mutexes. It's + * governed by the requirement that @ref heap_ready may block indefinitely, and + * this must not block other functions. Thus, several mutexes are involved: + * - @ref queue_mutex: protects values only used by @ref add_packet. This + * may be locked for long periods. + * - @ref config_mutex: protects configuration. The protected values are + * copied into @ref add_packet_state prior to adding a batch of packets. + * It is mostly locked for reads. + * - @ref stats_mutex: protects stream statistics, and is mostly locked for + * writes (assuming the user is only occasionally checking the stats). + * + * While holding @ref config_mutex or @ref stats_mutex it is illegal to lock + * any of the other mutexes. + * + * The public interface takes care of locking the appropriate mutexes. The + * private member functions generally expect the caller to take locks. */ class stream_base { +public: + struct add_packet_state; + private: struct queue_entry { @@ -146,40 +169,51 @@ * A particular heap is in a constructed state iff the next pointer is * not INVALID_ENTRY. */ - std::unique_ptr queue_storage; + const std::unique_ptr queue_storage; /// Number of entries in @ref buckets - std::size_t bucket_count; + const std::size_t bucket_count; /// Right shift to map 64-bit unsigned to a bucket index - int bucket_shift; + const int bucket_shift; /// Pointer to the first heap in each bucket, or NULL - std::unique_ptr buckets; + const std::unique_ptr buckets; /// Position of the most recently added heap std::size_t head; /// Maximum number of live heaps permitted. - std::size_t max_heaps; + const std::size_t max_heaps; /// Protocol bugs to be compatible with - bug_compat_mask bug_compat; + const bug_compat_mask bug_compat; - /// Function used to copy heap payloads - std::atomic memcpy{std::memcpy}; - /// Whether to stop when a stream control stop item is received - std::atomic stop_on_stop_item{true}; + /** + * Mutex protecting the state of the queue. This includes + * - @ref queue_storage + * - @ref buckets + * - @ref head + * - @ref stopped + */ + mutable std::mutex queue_mutex; - /// Mutex protecting @ref allocator - std::mutex allocator_mutex; /** - * Memory allocator used by heaps. - * - * This is protected by allocator_mutex. C++11 mandates free @c atomic_load - * and @c atomic_store on @c shared_ptr, but GCC 4.8 doesn't implement it. - * Also, std::atomic> causes undefined symbol errors, and - * is illegal because shared_ptr is not a POD type. + * Mutex protecting configuration. This includes + * - @ref allocator + * - @ref memcpy + * - @ref stop_on_stop_item + * - @ref allow_unsized_heaps */ + mutable std::mutex config_mutex; + + /// Function used to copy heap payloads + packet_memcpy_function memcpy; + /// Whether to stop when a stream control stop item is received + bool stop_on_stop_item = true; + /// Whether to permit packets that don't have HEAP_LENGTH item + bool allow_unsized_heaps = true; + + /// Memory allocator used by heaps. std::shared_ptr allocator; /// @ref stop_received has been called, either externally or by stream control - std::atomic stopped{false}; + bool stopped = false; /// Compute bucket number for a heap cnt std::size_t get_bucket(s_item_pointer_t heap_cnt) const; @@ -197,28 +231,52 @@ /** * Callback called when a heap is being ejected from the live list. - * The heap might or might not be complete. + * The heap might or might not be complete. The @ref queue_mutex will be + * locked during this call, which will block @ref stop and @ref flush. */ virtual void heap_ready(live_heap &&) {} + /// Implementation of @ref flush that assumes the caller has locked @ref queue_mutex + void flush_unlocked(); + + /// Implementation of @ref stop that assumes the caller has locked @ref queue_mutex + void stop_unlocked(); + + /// Implementation of @ref add_packet_state::add_packet + bool add_packet(add_packet_state &state, const packet_header &packet); + protected: mutable std::mutex stats_mutex; stream_stats stats; + /** + * Shut down the stream. This calls @ref flush_unlocked. Subclasses may + * override this to achieve additional effects, but must chain to the base + * implementation. It is guaranteed that it will only be called once. + * + * It is undefined what happens if @ref add_packet is called after a stream + * is stopped. + * + * This is called with @ref queue_mutex locked. Users must not call this + * function themselves; instead, call @ref stop. + */ + virtual void stop_received(); + public: /** - * State for a batch of calls to @ref add_packet. The strand must be held for - * its entire lifetime. It holds copies of data that otherwise require - * atomic access in the owning class. + * State for a batch of calls to @ref add_packet. Constructing this object + * locks the stream's @ref queue_mutex. */ struct add_packet_state { stream_base &owner; + std::lock_guard lock; ///< Holds a lock on the owner's @ref queue_mutex // Copied from the stream, but unencumbered by locks/atomics - memcpy_function memcpy; + packet_memcpy_function memcpy; std::shared_ptr allocator; bool stop_on_stop_item; + bool allow_unsized_heaps; // Updates to the statistics std::uint64_t packets = 0; std::uint64_t complete_heaps = 0; @@ -228,6 +286,19 @@ explicit add_packet_state(stream_base &owner); ~add_packet_state(); + + bool is_stopped() const { return owner.stopped; } + /// Indicate that the stream has stopped (e.g. because the remote peer disconnected) + void stop() { owner.stop_unlocked(); } + /** + * Add a packet that was received, and which has been examined by @ref + * decode_packet, and returns @c true if it is consumed. Even though @ref + * decode_packet does some basic sanity-checking, it may still be rejected + * by @ref live_heap::add_packet e.g., because it is a duplicate. + * + * It is an error to call this after the stream has been stopped. + */ + bool add_packet(const packet_header &packet) { return owner.add_packet(*this, packet); } }; static constexpr std::size_t default_max_heaps = 4; @@ -254,6 +325,9 @@ void set_memory_allocator(std::shared_ptr allocator); /// Set an alternative memcpy function for copying heap payload + void set_memcpy(packet_memcpy_function memcpy); + + /// Set an alternative memcpy function for copying heap payload void set_memcpy(memcpy_function memcpy); /// Set builtin memcpy function to use for copying payload @@ -265,44 +339,32 @@ /// Get whether to stop the stream when a stop item is received bool get_stop_on_stop_item() const; - /** - * Add a packet that was received, and which has been examined by @a - * decode_packet, and returns @c true if it is consumed. Even though @a - * decode_packet does some basic sanity-checking, it may still be rejected - * by @ref live_heap::add_packet e.g., because it is a duplicate. - * - * It is an error to call this after the stream has been stopped. - */ - bool add_packet(add_packet_state &state, const packet_header &packet); - - /** - * Shut down the stream. This calls @ref flush. Subclasses may override - * this to achieve additional effects, but must chain to the base - * implementation. - * - * It is undefined what happens if @ref add_packet is called after a stream - * is stopped. - */ - virtual void stop_received(); + /// Set whether to allow heaps without HEAP_LENGTH + void set_allow_unsized_heaps(bool allow); - bool is_stopped() const { return stopped.load(); } + /// Get whether to allow heaps without HEAP_LENGTH + bool get_allow_unsized_heaps() const; bug_compat_mask get_bug_compat() const { return bug_compat; } /// Flush the collection of live heaps, passing them to @ref heap_ready. void flush(); + + /** + * Stop the stream. This calls @ref stop_received. + */ + void stop(); + + /** + * Return statistics about the stream. See the Python documentation. + */ + stream_stats get_stats() const; }; /** - * Stream that is fed by subclasses of @ref reader. Unless otherwise specified, - * methods in @ref stream_base may only be called while holding the strand - * contained in this class. The public interface functions must be called - * from outside the strand (and outside the threads associated with the - * io_service), but are not thread-safe relative to each other. - * - * This class is thread-safe. This is achieved mostly by having operations run - * as completion handlers on a strand. The exception is @ref stop, which uses a - * @c once to ensure that only the first call actually runs. + * Stream that is fed by subclasses of @ref reader. + * + * The public interface to this class is thread-safe. */ class stream : protected stream_base { @@ -312,32 +374,27 @@ /// Holder that just ensures that the thread pool doesn't vanish std::shared_ptr thread_pool_holder; - /** - * Serialization of access. - */ - boost::asio::io_service::strand strand; + /// I/O service used by the readers + boost::asio::io_service &io_service; + + /// Protects mutable state (@ref readers, @ref stop_readers, @ref lossy). + mutable std::mutex reader_mutex; /** * Readers providing the stream data. */ std::vector > readers; + /// Set to true to indicate that no new readers should be added + bool stop_readers = false; + + /// True if any lossy reader has been added + bool lossy = false; + /// Ensure that @ref stop is only run once std::once_flag stop_once; - template - void emplace_reader_callback(Args&&... args) - { - if (!is_stopped()) - { - // Guarantee space before constructing the reader - readers.emplace_back(nullptr); - readers.pop_back(); - std::unique_ptr ptr(reader_factory::make_reader(*this, std::forward(args)...)); - if (ptr->lossy()) - lossy = true; - readers.push_back(std::move(ptr)); - } - } + /// Incremented by readers when they die + semaphore readers_stopped; /* Prevent moving (copying is already impossible). Moving is not safe * because readers refer back to *this (it could potentially be added if @@ -350,34 +407,6 @@ protected: virtual void stop_received() override; - /** - * Schedule execution of the function object @a callback through the @c - * io_service using the strand, and block until it completes. If the - * function throws an exception, it is rethrown in this thread. - */ - template - typename std::result_of::type run_in_strand(F &&func) - { - typedef typename std::result_of::type return_type; - std::packaged_task task(std::forward(func)); - auto future = task.get_future(); - get_strand().dispatch([&task] - { - /* This is subtle: task lives on the run_in_strand stack frame, so - * we have to be very careful not to touch it after that function - * exits. Calling task() directly can continue to touch task even - * after it has unblocked the future. But the move constructor for - * packaged_task will take over the shared state for the future. - */ - std::packaged_task my_task(std::move(task)); - my_task(); - }); - return future.get(); - } - - /// True if any lossy reader has been added (only access with strand held) - bool lossy; - /// Actual implementation of @ref stop void stop_impl(); @@ -389,12 +418,15 @@ using stream_base::set_memcpy; using stream_base::set_stop_on_stop_item; using stream_base::get_stop_on_stop_item; - - boost::asio::io_service::strand &get_strand() { return strand; } + using stream_base::set_allow_unsized_heaps; + using stream_base::get_allow_unsized_heaps; + using stream_base::get_stats; explicit stream(io_service_ref io_service, bug_compat_mask bug_compat = 0, std::size_t max_heaps = default_max_heaps); virtual ~stream() override; + boost::asio::io_service &get_io_service() { return io_service; } + /** * Add a new reader by passing its constructor arguments, excluding * the initial @a stream argument. @@ -402,28 +434,33 @@ template void emplace_reader(Args&&... args) { - // This would probably work better with a lambda (better forwarding), - // but GCC 4.8 has a bug with accessing parameter packs inside a - // lambda. - run_in_strand(detail::reference_bind( - std::mem_fn(&stream::emplace_reader_callback), - this, std::forward(args)...)); + std::lock_guard lock(reader_mutex); + // See comments in stop_impl for why we do this check + if (!stop_readers) + { + // Guarantee space before constructing the reader + readers.emplace_back(nullptr); + readers.pop_back(); + std::unique_ptr ptr(reader_factory::make_reader(*this, std::forward(args)...)); + if (ptr->lossy()) + lossy = true; + readers.push_back(std::move(ptr)); + } } /** - * Return statistics about the stream. See the Python documentation. - */ - stream_stats get_stats() const; - - /** * Stop the stream and block until all the readers have wound up. After * calling this there should be no more outstanding completion handlers * in the thread pool. * * In most cases subclasses should override @ref stop_received rather than - * this function. + * this function. However, if @ref heap_ready can block indefinitely, this + * function should be overridden to unblock it before calling the base + * implementation. */ virtual void stop(); + + bool is_lossy() const; }; /** diff -Nru spead2-1.10.0/include/spead2/recv_tcp.h spead2-2.1.0/include/spead2/recv_tcp.h --- spead2-1.10.0/include/spead2/recv_tcp.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_tcp.h 2019-03-07 07:46:37.000000000 +0000 @@ -76,13 +76,13 @@ std::size_t bytes_transferred); /// Processes the content of the buffer, returns true if more reading needs to be enqueued - bool process_buffer(const std::size_t bytes_recv); + bool process_buffer(stream_base::add_packet_state &state, const std::size_t bytes_recv); /// Parses the size of the next packet to read from the stream, returns true if more data needs to be read to parse the packet size correctly bool parse_packet_size(); /// Parses the next packet out of the stream, returns false if the contents of the current stream are not enough - bool parse_packet(); + bool parse_packet(stream_base::add_packet_state &state); /// Ignores bytes from the stream according to @a to_skip, returns true if more data needs to be read and skipped bool skip_bytes(); diff -Nru spead2-1.10.0/include/spead2/recv_udp_base.h spead2-2.1.0/include/spead2/recv_udp_base.h --- spead2-1.10.0/include/spead2/recv_udp_base.h 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_udp_base.h 2019-03-06 09:53:45.000000000 +0000 @@ -24,6 +24,7 @@ #include #include #include +#include namespace spead2 { diff -Nru spead2-1.10.0/include/spead2/recv_udp.h spead2-2.1.0/include/spead2/recv_udp.h --- spead2-1.10.0/include/spead2/recv_udp.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_udp.h 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -53,14 +53,6 @@ /// Maximum packet size we will accept std::size_t max_size; #if SPEAD2_USE_RECVMMSG - /** - * A dup(2) of @ref socket. This is used for the actual recvmmsg call. The - * duplicate is needed so that we can asynchronously call socket.close() - * to shut down the reader while racing with the recvmmsg call. The - * close call will just cancel pending handlers, without causing us to - * read from a closed file descriptor. - */ - boost::asio::ip::udp::socket socket2; /// Buffer for asynchronous receive, of size @a max_size + 1. std::vector> buffer; /// Scatter-gather array for each buffer @@ -107,20 +99,26 @@ std::size_t buffer_size = default_buffer_size); /** - * Constructor with explicit multicast interface address (IPv4 only). + * Constructor with explicit interface address (IPv4 only). + * + * This overload is designed for use with multicast, but can also be used + * with a unicast endpoint as long as the address matches the interface + * address. * - * The socket will have @c SO_REUSEADDR set, so that multiple sockets can - * all listen to the same multicast stream. If you want to let the - * system pick the interface for the multicast subscription, use - * @c boost::asio::ip::address_v4::any(), or use the default constructor. + * When a multicast group is used, the socket will have @c SO_REUSEADDR + * set, so that multiple sockets can all listen to the same multicast + * stream. If you want to let the system pick the interface for the + * multicast subscription, use @c boost::asio::ip::address_v4::any(), or + * use the default constructor. * * @param owner Owning stream - * @param endpoint Multicast group and port + * @param endpoint Address and port * @param max_size Maximum packet size that will be accepted. * @param buffer_size Requested socket buffer size. * @param interface_address Address of the interface which should join the group * - * @throws std::invalid_argument If @a endpoint is not an IPv4 multicast address + * @throws std::invalid_argument If @a endpoint is not an IPv4 multicast address and + * does not match @a interface_address. * @throws std::invalid_argument If @a interface_address is not an IPv4 address */ udp_reader( diff -Nru spead2-1.10.0/include/spead2/recv_udp_ibv.h spead2-2.1.0/include/spead2/recv_udp_ibv.h --- spead2-1.10.0/include/spead2/recv_udp_ibv.h 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_udp_ibv.h 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016 SKA South Africa +/* Copyright 2016, 2019-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -32,10 +32,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -45,106 +47,236 @@ namespace recv { -/** - * Synchronous or asynchronous stream reader that reads UDP packets using - * the Infiniband verbs API. It currently only supports multicast IPv4, with - * no fragmentation, IP header options, or VLAN tags. - */ -class udp_ibv_reader : public udp_reader_base +namespace detail { -private: - struct slot : boost::noncopyable - { - ibv_recv_wr wr; - ibv_sge sge; - }; - - ///< Maximum supported packet size - const std::size_t max_size; - ///< Number of packets that can be queued - const std::size_t n_slots; - ///< Number of times to poll before waiting - const int max_poll; +/* Parts of udp_ibv_reader_base that don't need to be templated */ +class udp_ibv_reader_core : public udp_reader_base +{ +private: /** * Socket that is used only to join the multicast group. It is not * bound to a port. */ boost::asio::ip::udp::socket join_socket; - /// Data buffer for all the packets - memory_allocator::pointer buffer; - // All the data structures required by ibverbs +protected: + enum class poll_result + { + stopped, ///< stream was stopped + partial, ///< read zero or more CQEs, but stopped to prevent livelock + drained, ///< CQ fully drained + }; + + // Data structures required by ibverbs rdma_event_channel_t event_channel; rdma_cm_id_t cm_id; ibv_pd_t pd; ibv_comp_channel_t comp_channel; boost::asio::posix::stream_descriptor comp_channel_wrapper; - ibv_cq_t send_cq; - ibv_cq_t recv_cq; - ibv_qp_t qp; std::vector flows; - ibv_mr_t mr; + ibv_cq_t recv_cq; - /// array of @ref n_slots slots for work requests - std::unique_ptr slots; - /// array of @ref n_slots work completions - std::unique_ptr wc; + ///< Maximum supported packet size + const std::size_t max_size; + ///< Number of times to poll before waiting + const int max_poll; /// Signals poll-mode to stop std::atomic stop_poll; - // Utility functions to create the data structures - static ibv_qp_t - create_qp(const ibv_pd_t &pd, const ibv_cq_t &send_cq, const ibv_cq_t &recv_cq, - std::size_t n_slots); - - static ibv_flow_t - create_flow(const ibv_qp_t &qp, const boost::asio::ip::udp::endpoint &endpoint, - int port_num); + void join_groups(const std::vector &endpoints, + const boost::asio::ip::address &interface_address); - static void req_notify_cq(ibv_cq *cq); +public: + /// Receive buffer size, if none is explicitly passed to the constructor + static constexpr std::size_t default_buffer_size = 16 * 1024 * 1024; + /// Number of times to poll in a row, if none is explicitly passed to the constructor + static constexpr int default_max_poll = 10; - /** - * Do one pass over the completion queue. - * - * @retval -1 if there was an ibverbs failure - * @retval -2 if the stream received a stop packet - * @retval n otherwise, where n is the number of packets received - */ - int poll_once(stream_base::add_packet_state &state); + udp_ibv_reader_core( + stream &owner, + const std::vector &endpoints, + const boost::asio::ip::address &interface_address, + std::size_t max_size, + int comp_vector, + int max_poll); + + virtual void stop() override; +}; +/** + * Common code between spead2::recv::udp_ibv_reader and + * spead2::recv::udp_ibv_mprq_reader. It uses the curiously recursive template + * pattern to avoid virtual functions. + */ +template +class udp_ibv_reader_base : public udp_ibv_reader_core +{ +protected: /** * Retrieve packets from the completion queue and process them. * * This is called from the io_service either when the completion channel - * is notified (non-polling mode) or by a post to the strand (polling + * is notified (non-polling mode) or by a post to the io_service (polling * mode). + * + * If @a consume_event is true, an event should be removed and consumed + * from the completion channel. */ - void packet_handler(const boost::system::error_code &error); + void packet_handler(const boost::system::error_code &error, + bool consume_event); /** * Request a callback when there is data (or as soon as possible, in - * polling mode). + * polling mode or when @a need_poll is true). */ - void enqueue_receive(); + void enqueue_receive(bool needs_poll); -public: - /// Receive buffer size, if none is explicitly passed to the constructor - static constexpr std::size_t default_buffer_size = 16 * 1024 * 1024; - /// Number of times to poll in a row, if none is explicitly passed to the constructor - static constexpr int default_max_poll = 10; + using udp_ibv_reader_core::udp_ibv_reader_core; +}; + +template +void udp_ibv_reader_base::packet_handler(const boost::system::error_code &error, + bool consume_event) +{ + stream_base::add_packet_state state(get_stream_base()); + + bool need_poll = true; + if (!error) + { + if (consume_event) + { + ibv_cq *event_cq; + void *event_context; + while (comp_channel.get_event(&event_cq, &event_context)) + { + // TODO: defer acks until shutdown + recv_cq.ack_events(1); + } + } + if (state.is_stopped()) + { + log_info("UDP reader: discarding packet received after stream stopped"); + } + else + { + for (int i = 0; i < max_poll; i++) + { + if (comp_channel) + { + if (i == max_poll - 1) + { + /* We need to call req_notify_cq *before* the last + * poll_once, because notifications are edge-triggered. + * If we did it the other way around, there is a race + * where a new packet can arrive after poll_once but + * before req_notify_cq, failing to trigger a + * notification. + */ + recv_cq.req_notify(false); + need_poll = false; + } + } + else if (stop_poll.load()) + break; + poll_result result = static_cast(this)->poll_once(state); + if (result == poll_result::stopped) + break; + else if (result == poll_result::partial) + { + /* If we armed req_notify_cq but then didn't drain the CQ, and + * we get no more packets, then we won't get woken up again, so + * we need to poll again next time we go around the event loop. + */ + need_poll = true; + } + } + } + } + else if (error != boost::asio::error::operation_aborted) + log_warning("Error in UDP receiver: %1%", error.message()); + + if (!state.is_stopped()) + { + enqueue_receive(need_poll); + } + else + stopped(); +} + +template +void udp_ibv_reader_base::enqueue_receive(bool need_poll) +{ + using namespace std::placeholders; + if (comp_channel && !need_poll) + { + // Asynchronous mode + comp_channel_wrapper.async_read_some( + boost::asio::null_buffers(), + std::bind(&udp_ibv_reader_base::packet_handler, this, _1, true)); + } + else + { + // Polling mode + get_io_service().post( + std::bind(&udp_ibv_reader_base::packet_handler, this, + boost::system::error_code(), false)); + } +} + +} // namespace detail + +/** + * Synchronous or asynchronous stream reader that reads UDP packets using + * the Infiniband verbs API. It currently only supports IPv4, with + * no fragmentation, IP header options, or VLAN tags. + */ +class udp_ibv_reader : public detail::udp_ibv_reader_base +{ +private: + friend class detail::udp_ibv_reader_base; + + struct slot : boost::noncopyable + { + ibv_recv_wr wr; + ibv_sge sge; + }; + + // All the data structures required by ibverbs + ibv_cq_t send_cq; + ibv_qp_t qp; + ibv_mr_t mr; + ///< Number of packets that can be queued + const std::size_t n_slots; + + /// Data buffer for all the packets + memory_allocator::pointer buffer; + + /// array of @ref n_slots slots for work requests + std::unique_ptr slots; + /// array of @ref n_slots work completions + std::unique_ptr wc; + + /// Do one pass over the completion queue. + poll_result poll_once(stream_base::add_packet_state &state); + +public: /** * Constructor. * * @param owner Owning stream - * @param endpoint Multicast group and port + * @param endpoint Address and port. Note that is it possible for the address to be + * unicast and different to the @a interface_address: the interface + * may have multiple IP addresses, in which case this filters packets + * on the interface by IP address. An unspecified address can also be + * used to skip address filtering. + * @param interface_address Address of the interface which should join the group and listen for data * @param max_size Maximum packet size that will be accepted * @param buffer_size Requested memory allocation for work requests. Note * that this is used to determine the number of packets * to buffer; if the packets are smaller than @a max_size, * then fewer bytes will be buffered. - * @param interface_address Address of the interface which should join the group and listen for data * @param comp_vector Completion channel vector (interrupt) for asynchronous operation, or * a negative value to poll continuously. Polling * should not be used if there are other users of the @@ -159,7 +291,8 @@ * non-negative) or letting other code run on the * thread (if @a comp_vector is negative). * - * @throws std::invalid_argument If @a endpoint is not an IPv4 multicast address + * @throws std::invalid_argument If @a endpoint is specified and is not an + * IPv4 address * @throws std::invalid_argument If @a interface_address is not an IPv4 address */ udp_ibv_reader( @@ -175,13 +308,17 @@ * Constructor with multiple endpoints. * * @param owner Owning stream - * @param endpoints Multicast groups and ports + * @param endpoints Addresses and ports. Note that is it possible for the addresses to be + * unicast and different to the @a interface_address: the interface + * may have multiple IP addresses, in which case this filters packets + * on the interface by IP address. An unspecified address can also be + * used to skip address filtering. + * @param interface_address Address of the interface which should join the group and listen for data * @param max_size Maximum packet size that will be accepted * @param buffer_size Requested memory allocation for work requests. Note * that this is used to determine the number of packets * to buffer; if the packets are smaller than @a max_size, * then fewer bytes will be buffered. - * @param interface_address Address of the interface which should join the group and listen for data * @param comp_vector Completion channel vector (interrupt) for asynchronous operation, or * a negative value to poll continuously. Polling * should not be used if there are other users of the @@ -196,7 +333,8 @@ * non-negative) or letting other code run on the * thread (if @a comp_vector is negative). * - * @throws std::invalid_argument If any element of @a endpoints is not an IPv4 multicast address + * @throws std::invalid_argument If any element of @a endpoints is specified and is not + * an IPv4 address * @throws std::invalid_argument If @a interface_address is not an IPv4 address */ udp_ibv_reader( @@ -207,8 +345,49 @@ std::size_t buffer_size = default_buffer_size, int comp_vector = 0, int max_poll = default_max_poll); +}; - virtual void stop() override; +} // namespace recv +} // namespace spead2 + +#include + +namespace spead2 +{ +namespace recv +{ + +template<> +struct reader_factory +{ + template + static std::unique_ptr make_reader(Args&&... args) + { + /* Note: using perfect forwarding twice on the same args is normally a + * bad idea if any of them are rvalue references. But the constructors + * we're forwarding to don't have any. + */ +#if SPEAD2_USE_IBV_MPRQ + try + { + std::unique_ptr reader(new udp_ibv_mprq_reader( + std::forward(args)...)); + log_info("Using multi-packet receive queue for verbs acceleration"); + return reader; + } + catch (std::system_error &e) + { + if (e.code() != std::errc::not_supported) + throw; + log_debug("Multi-packet receive queues not supported (%1%), falling back", e.what()); + return std::unique_ptr(new udp_ibv_reader( + std::forward(args)...)); + } +#else + return std::unique_ptr(new udp_ibv_reader( + std::forward(args)...)); +#endif + } }; } // namespace recv diff -Nru spead2-1.10.0/include/spead2/recv_udp_ibv_mprq.h spead2-2.1.0/include/spead2/recv_udp_ibv_mprq.h --- spead2-1.10.0/include/spead2/recv_udp_ibv_mprq.h 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/include/spead2/recv_udp_ibv_mprq.h 2019-01-14 12:03:16.000000000 +0000 @@ -0,0 +1,165 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + */ + +#ifndef SPEAD2_RECV_UDP_IBV_MPRQ_H +#define SPEAD2_RECV_UDP_IBV_MPRQ_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include +#if SPEAD2_USE_IBV_MPRQ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ +namespace recv +{ + +/** + * Synchronous or asynchronous stream reader that reads UDP packets using + * the Infiniband verbs API with multi-packet receive queues. It currently only + * supports multicast IPv4, with no fragmentation, IP header options, or VLAN + * tags. + */ +class udp_ibv_mprq_reader : public detail::udp_ibv_reader_base +{ +private: + friend class detail::udp_ibv_reader_base; + + // All the data structures required by ibverbs + ibv_exp_res_domain_t res_domain; + ibv_exp_wq_t wq; + ibv_exp_rwq_ind_table_t rwq_ind_table; + ibv_exp_cq_family_v1_t cq_intf; + ibv_exp_wq_family_t wq_intf; + ibv_qp_t qp; + ibv_mr_t mr; + + /// Data buffer for all the packets + memory_allocator::pointer buffer; + + /// Bytes of buffer for each WQ entry + std::size_t wqe_size; + /// Buffer offset of the current WQE + std::size_t wqe_start = 0; + /// Total buffer size + std::size_t buffer_size; + + /// Post one work request to the receive work queue + void post_wr(std::size_t offset); + + /// Do one pass over the completion queue. + poll_result poll_once(stream_base::add_packet_state &state); + +public: + /** + * Constructor. + * + * @param owner Owning stream + * @param endpoint Multicast group and port + * @param max_size Maximum packet size that will be accepted + * @param buffer_size Requested memory allocation for work requests. Note + * that this is used to determine the number of packets + * to buffer; if the packets are smaller than @a max_size, + * then fewer bytes will be buffered. + * @param interface_address Address of the interface which should join the group and listen for data + * @param comp_vector Completion channel vector (interrupt) for asynchronous operation, or + * a negative value to poll continuously. Polling + * should not be used if there are other users of the + * thread pool. If a non-negative value is provided, it + * is taken modulo the number of available completion + * vectors. This allows a number of readers to be + * assigned sequential completion vectors and have them + * load-balanced, without concern for the number + * available. + * @param max_poll Maximum number of times to poll in a row, without + * waiting for an interrupt (if @a comp_vector is + * non-negative) or letting other code run on the + * thread (if @a comp_vector is negative). + * + * @throws std::invalid_argument If @a endpoint is not an IPv4 multicast address + * @throws std::invalid_argument If @a interface_address is not an IPv4 address + */ + udp_ibv_mprq_reader( + stream &owner, + const boost::asio::ip::udp::endpoint &endpoint, + const boost::asio::ip::address &interface_address, + std::size_t max_size = default_max_size, + std::size_t buffer_size = default_buffer_size, + int comp_vector = 0, + int max_poll = default_max_poll); + + /** + * Constructor with multiple endpoints. + * + * @param owner Owning stream + * @param endpoints Multicast groups and ports + * @param max_size Maximum packet size that will be accepted + * @param buffer_size Requested memory allocation for work requests. Note + * that this is used to determine the number of packets + * to buffer; if the packets are smaller than @a max_size, + * then fewer bytes will be buffered. + * @param interface_address Address of the interface which should join the group and listen for data + * @param comp_vector Completion channel vector (interrupt) for asynchronous operation, or + * a negative value to poll continuously. Polling + * should not be used if there are other users of the + * thread pool. If a non-negative value is provided, it + * is taken modulo the number of available completion + * vectors. This allows a number of readers to be + * assigned sequential completion vectors and have them + * load-balanced, without concern for the number + * available. + * @param max_poll Maximum number of times to poll in a row, without + * waiting for an interrupt (if @a comp_vector is + * non-negative) or letting other code run on the + * thread (if @a comp_vector is negative). + * + * @throws std::invalid_argument If any element of @a endpoints is not an IPv4 multicast address + * @throws std::invalid_argument If @a interface_address is not an IPv4 address + */ + udp_ibv_mprq_reader( + stream &owner, + const std::vector &endpoints, + const boost::asio::ip::address &interface_address, + std::size_t max_size = default_max_size, + std::size_t buffer_size = default_buffer_size, + int comp_vector = 0, + int max_poll = default_max_poll); +}; + +} // namespace recv +} // namespace spead2 + +#endif // SPEAD2_USE_IBV + +#endif // SPEAD2_RECV_UDP_IBV_H diff -Nru spead2-1.10.0/include/spead2/send_heap.h spead2-2.1.0/include/spead2/send_heap.h --- spead2-1.10.0/include/spead2/send_heap.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_heap.h 2019-03-06 09:53:45.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -126,6 +126,7 @@ friend class packet_generator; private: flavour flavour_; + bool repeat_pointers = false; /// Items to write (including descriptors) std::vector items; @@ -222,6 +223,29 @@ { add_item(STREAM_CTRL_ID, CTRL_STREAM_STOP); } + + /** + * Enable/disable repetition of item pointers in all packets. + * + * Usually this is not needed, but it can enable some specialised use + * cases where immediates can be recovered from incomplete heaps or where + * the receiver examines the item pointers in each packet to decide how + * to handle it. The packet size must be large enough to fit all the item + * pointers for the heap (the implementation also reserves a little space, + * so do not rely on a tight fit working). + * + * The default is disabled. + */ + void set_repeat_pointers(bool repeat) + { + repeat_pointers = repeat; + } + + /// Return the flag set by @ref set_repeat_pointers. + bool get_repeat_pointers() const + { + return repeat_pointers; + } }; } // namespace send diff -Nru spead2-1.10.0/include/spead2/send_inproc.h spead2-2.1.0/include/spead2/send_inproc.h --- spead2-1.10.0/include/spead2/send_inproc.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_inproc.h 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2018 SKA South Africa +/* Copyright 2018, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -48,25 +48,10 @@ { private: friend class stream_impl; - - template - void async_send_packet(const packet &pkt, Handler &&handler) - { - inproc_queue::packet dup = detail::copy_packet(pkt); - std::size_t size = dup.size; - try - { - queue->buffer.push(std::move(dup)); - get_io_service().post(std::bind(handler, boost::system::error_code(), size)); - } - catch (ringbuffer_stopped) - { - get_io_service().post(std::bind(handler, boost::asio::error::operation_aborted, 0)); - } - } - std::shared_ptr queue; + void async_send_packets(); + public: /// Constructor inproc_stream( diff -Nru spead2-1.10.0/include/spead2/send_packet.h spead2-2.1.0/include/spead2/send_packet.h --- spead2-1.10.0/include/spead2/send_packet.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_packet.h 2019-05-13 09:15:58.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -80,6 +80,7 @@ public: packet_generator(const heap &h, item_pointer_t cnt, std::size_t max_packet_size); + bool has_next_packet() const; packet next_packet(); }; diff -Nru spead2-1.10.0/include/spead2/send_streambuf.h spead2-2.1.0/include/spead2/send_streambuf.h --- spead2-1.10.0/include/spead2/send_streambuf.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_streambuf.h 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -42,19 +42,7 @@ friend class stream_impl; std::streambuf &streambuf; - template - void async_send_packet(const packet &pkt, Handler &&handler) - { - std::size_t size = 0; - for (const auto &buffer : pkt.buffers) - { - std::size_t buffer_size = boost::asio::buffer_size(buffer); - // TODO: handle errors - streambuf.sputn(boost::asio::buffer_cast(buffer), buffer_size); - size += buffer_size; - } - get_io_service().dispatch(std::bind(std::move(handler), boost::system::error_code(), size)); - } + void async_send_packets(); public: /// Constructor diff -Nru spead2-1.10.0/include/spead2/send_stream.h spead2-2.1.0/include/spead2/send_stream.h --- spead2-1.10.0/include/spead2/send_stream.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_stream.h 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -25,14 +25,17 @@ #include #include #include -#include #include #include +#include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -54,15 +57,15 @@ static constexpr double default_burst_rate_ratio = 1.05; void set_max_packet_size(std::size_t max_packet_size); - std::size_t get_max_packet_size() const; + std::size_t get_max_packet_size() const { return max_packet_size; } void set_rate(double rate); - double get_rate() const; + double get_rate() const { return rate; } void set_burst_size(std::size_t burst_size); - std::size_t get_burst_size() const; + std::size_t get_burst_size() const { return burst_size; } void set_max_heaps(std::size_t max_heaps); - std::size_t get_max_heaps() const; + std::size_t get_max_heaps() const { return max_heaps; } void set_burst_rate_ratio(double burst_rate_ratio); - double get_burst_rate_ratio() const; + double get_burst_rate_ratio() const { return burst_rate_ratio; } /// Get product of rate and burst_rate_ratio double get_burst_rate() const; @@ -148,15 +151,25 @@ virtual ~stream(); }; +template +class stream_impl; + /** - * Stream that sends packets at a maximum rate. It also serialises heaps so - * that only one heap is being sent at a time. Heaps are placed in a queue, and if - * the queue becomes too long heaps are discarded. + * Base class for @ref stream_impl. It implements the parts that are + * independent of the transport. */ -template -class stream_impl : public stream +class stream_impl_base : public stream { + template friend class stream_impl; private: + enum class state_t + { + QUEUED, + SENDING, + SLEEPING, + EMPTY + }; + typedef boost::asio::basic_waitable_timer timer_type; struct queue_item @@ -164,211 +177,266 @@ const heap &h; item_pointer_t cnt; completion_handler handler; + item_pointer_t bytes_sent = 0; queue_item() = default; - queue_item(const heap &h, item_pointer_t cnt, completion_handler &&handler) + queue_item(const heap &h, item_pointer_t cnt, completion_handler &&handler) noexcept : h(std::move(h)), cnt(cnt), handler(std::move(handler)) { } }; + typedef std::aligned_storage::type queue_item_storage; + +protected: + struct transmit_packet + { + packet pkt; + std::size_t size; + bool last; // if this is the last packet in the heap + queue_item *item; + boost::system::error_code result; + }; + + std::unique_ptr current_packets; + std::size_t n_current_packets = 0; + const std::size_t max_current_packets; + +private: const stream_config config; const double seconds_per_byte_burst, seconds_per_byte; /** - * Protects access to @a queue. All other members are either const or are - * only accessed only by completion handlers, and there is only ever one - * scheduled at a time. + * Protects access to + * - @ref queue_head and @ref queue_tail + * - @ref state + * - @ref next_cnt and @ref step_cnt */ std::mutex queue_mutex; - std::queue queue; + /** + * Circular queue with config.max_heaps + 1 slots, which must never be full + * (because that can't be distinguished from empty). Items from @ref + * queue_head to @ref queue_tail are constructed in place, while the rest + * is uninitialised raw storage. + */ + std::unique_ptr queue; + /// Bounds of the queue + std::size_t queue_head = 0, queue_tail = 0; + /// Item holding data for the next packet to send + std::size_t active = 0; // queue slot for next packet to send + state_t state = state_t::EMPTY; timer_type timer; - // Time at which next burst should be sent, considering the burst rate + /// Time at which next burst should be sent, considering the burst rate timer_type::time_point send_time_burst; - // Time at which next burst should be sent, considering the average rate + /// Time at which next burst should be sent, considering the average rate timer_type::time_point send_time; - /// Number of bytes sent in the current heap - item_pointer_t heap_bytes = 0; /// Number of bytes sent since send_time and sent_time_burst were updated std::uint64_t rate_bytes = 0; /// Heap cnt for the next heap to send item_pointer_t next_cnt = 1; /// Increment to next_cnt after each heap item_pointer_t step_cnt = 1; - std::unique_ptr gen; // TODO: make this inlinable - /// Packet undergoing transmission by send_next_packet - packet current_packet; - /// Signalled whenever the last heap is popped from the queue + /** + * Packet generator for the active heap. It may be empty at any time, which + * indicates that it should be initialised from the heap indicated by + * @ref active. + * + * When non-empty, it must always have a next packet i.e. after + * exhausting it, it must be cleared/changed. + */ + boost::optional gen; + /// Signalled when transitioning to EMPTY state std::condition_variable heap_empty; + /// Get next slot position in queue + std::size_t next_queue_slot(std::size_t cur) const; + + /// Access an item from the queue + queue_item *get_queue(std::size_t idx); + + /** + * Advance @ref active to the next heap and clear @ref gen. + * It does not need @ref queue_mutex. + */ + void next_active(); + /** - * Asynchronously send the next packet from the current heap - * (or the next heap, if the current one is finished). + * Set the result of the first heap in the queue and remove it. This must be + * called with @ref queue_mutex held. + */ + void post_handler(boost::system::error_code result); + + /** + * Whether a full burst has been transmitted, requiring some sleep time. + * Does not require @ref queue_mutex. + */ + bool must_sleep() const; + + /** + * Apply per-packet transmission results to the queue. This must be called + * with @ref queue_mutex held. + */ + void process_results(); + + /** + * Update @ref send_time_burst and @ref send_time from @ref rate_bytes. + * Does not require @ref queue_mutex. * - * @param ec Error from sending the previous packet. If set, the rest of the - * current heap is aborted. + * @param now Current time + * @returns Time at which next packet should be sent + */ + timer_type::time_point update_send_times(timer_type::time_point now); + + /// Update @ref send_time after a period in state @c EMPTY. + void update_send_time_empty(); + + /** + * Populate @ref current_packets and @ref n_current_packets with packets to + * send. This is called without @ref queue_mutex held. It takes a copy of + * @ref queue_tail so that it does not need the lock to access the original. + */ + void load_packets(std::size_t tail); + +protected: + stream_impl_base(io_service_ref io_service, const stream_config &config, std::size_t max_current_packets); + virtual ~stream_impl_base() override; + +public: + virtual void set_cnt_sequence(item_pointer_t next, item_pointer_t step) override; + + /** + * Block until all enqueued heaps have been sent. This function is + * thread-safe, but can be live-locked if more heaps are added while it is + * running. + */ + virtual void flush() override; +}; + +/** + * Stream that sends packets at a maximum rate. It also serialises heaps so + * that only one heap is being sent at a time. Heaps are placed in a queue, and + * if the queue becomes too deep heaps are discarded. + * + * The stream operates as a state machine, depending on which handlers are + * pending: + * - QUEUED: was previously empty, but async_send_heap posted a callback to @ref do_next + * - SENDING: the derived class is in the process of sending packets + * - SLEEPING: we are sleeping as a result of rate limiting + * - EMPTY: there are no heaps and no pending callbacks. + * + * The derived class implements @c async_send_packets, which is responsible for + * arranging transmission of the @ref n_current_packets stored in + * @ref current_packets. Once the packets are sent, it must cause + * @ref packets_handler to be called (but not before returning). It may assume + * that there is at least one packet to send. + * + * There are two mechanisms to protect shared data: the @ref do_next callback + * is only ever scheduled once, so it does not need to lock data for which it + * is the only user. Data also accessed by user-facing functions are protected + * by a mutex. + */ +template +class stream_impl : public stream_impl_base +{ +private: + /** + * Advance the state machine. Whenever the state is not EMPTY, there is a + * future call to this function expected (possibly indirectly via + * @ref packets_handler). */ - void send_next_packet(boost::system::error_code ec = boost::system::error_code()) + void do_next() { - bool again; - do + std::unique_lock lock(queue_mutex); + if (state == state_t::SENDING) + process_results(); + else if (state == state_t::QUEUED) + update_send_time_empty(); + assert(active == queue_head); + + if (must_sleep()) { - assert(gen); - again = false; - current_packet = gen->next_packet(); - if (ec || current_packet.buffers.empty()) + auto now = timer_type::clock_type::now(); + auto target_time = update_send_times(now); + if (now < target_time) { - // Reached the end of a heap. Pop the current one, and start the - // next one if there is one. - completion_handler handler; - bool empty; - - gen.reset(); - std::unique_lock lock(queue_mutex); - handler = std::move(queue.front().handler); - queue.pop(); - empty = queue.empty(); - if (!empty) - gen.reset(new packet_generator( - queue.front().h, queue.front().cnt, config.get_max_packet_size())); - else - gen.reset(); - - std::size_t old_heap_bytes = heap_bytes; - heap_bytes = 0; - if (empty) - { - heap_empty.notify_all(); - // Avoid hanging on to data indefinitely - current_packet = packet(); - } - again = !empty; // Start again on the next heap - lock.unlock(); - - /* At this point it is not safe to touch *this at all, because - * if the queue is empty, the destructor is free to complete - * and take the memory out from under us. - */ - handler(ec, old_heap_bytes); + state = state_t::SLEEPING; + timer.expires_at(target_time); + timer.async_wait([this](const boost::system::error_code &) { do_next(); }); + return; } - else - { - auto send_next_callback = [this] (const boost::system::error_code &error) - { - send_next_packet(error); - }; - static_cast(this)->async_send_packet( - current_packet, - [this, send_next_callback] (const boost::system::error_code &ec, std::size_t bytes_transferred) - { - if (ec) - { - send_next_packet(ec); - return; - } - bool sleeping = false; - rate_bytes += bytes_transferred; - heap_bytes += bytes_transferred; - if (rate_bytes >= config.get_burst_size()) - { - auto now = timer_type::clock_type::now(); - std::chrono::duration wait_burst(rate_bytes * seconds_per_byte_burst); - std::chrono::duration wait(rate_bytes * seconds_per_byte); - send_time_burst += std::chrono::duration_cast(wait_burst); - send_time += std::chrono::duration_cast(wait); - /* send_time_burst needs to reflect the time the burst - * was actually sent (as well as we can estimate it), even if - * sent_time or now is later. - */ - auto target_time = max(send_time_burst, send_time); - send_time_burst = max(now, target_time); - rate_bytes = 0; - if (now < target_time) - { - sleeping = true; - timer.expires_at(target_time); - timer.async_wait(send_next_callback); - } - } - if (!sleeping) - send_next_packet(); - }); - } - } while (again); - } + } -public: - stream_impl( - io_service_ref io_service, - const stream_config &config = stream_config()) : - stream(std::move(io_service)), - config(config), - seconds_per_byte_burst(config.get_burst_rate() > 0.0 ? 1.0 / config.get_burst_rate() : 0.0), - seconds_per_byte(config.get_rate() > 0.0 ? 1.0 / config.get_rate() : 0.0), - timer(get_io_service()) - { + if (queue_head == queue_tail) + { + state = state_t::EMPTY; + heap_empty.notify_all(); + return; + } + + // Save a copy to use outside the protection of the lock. + std::size_t tail = queue_tail; + state = state_t::SENDING; + lock.unlock(); + + load_packets(tail); + assert(n_current_packets > 0); + static_cast(this)->async_send_packets(); } - virtual void set_cnt_sequence(item_pointer_t next, item_pointer_t step) override +protected: + /** + * Report on completed packets. Each element of @a items must have it's @c result + * field updated. + * + * This function must not be called directly from async_send_packets, as this will + * lead to a deadlock. It must be scheduled to be called later. + */ + void packets_handler() { - if (step == 0) - throw std::invalid_argument("step cannot be 0"); - std::unique_lock lock(queue_mutex); - next_cnt = next; - step_cnt = step; + do_next(); } + using stream_impl_base::stream_impl_base; + +public: virtual bool async_send_heap(const heap &h, completion_handler handler, s_item_pointer_t cnt = -1) override { std::unique_lock lock(queue_mutex); - if (queue.size() >= config.get_max_heaps()) + std::size_t new_tail = next_queue_slot(queue_tail); + if (new_tail == queue_head) { + lock.unlock(); log_warning("async_send_heap: dropping heap because queue is full"); - get_io_service().dispatch(std::bind(handler, boost::asio::error::would_block, 0)); + get_io_service().post(std::bind(handler, boost::asio::error::would_block, 0)); return false; } - bool empty = queue.empty(); - item_pointer_t ucnt; // unsigned, so that copying next_cnt cannot overflow + item_pointer_t cnt_mask = (item_pointer_t(1) << h.get_flavour().get_heap_address_bits()) - 1; if (cnt < 0) { - ucnt = next_cnt; + cnt = next_cnt & cnt_mask; next_cnt += step_cnt; } - else - ucnt = cnt; - queue.emplace(h, ucnt, std::move(handler)); - if (empty) + else if (item_pointer_t(cnt) > cnt_mask) { - assert(!gen); - gen.reset(new packet_generator(queue.front().h, queue.front().cnt, config.get_max_packet_size())); + lock.unlock(); + log_warning("async_send_heap: dropping heap because cnt is out of range"); + get_io_service().post(std::bind(handler, boost::asio::error::invalid_argument, 0)); + return false; } + + // Construct in place + new (get_queue(queue_tail)) queue_item(h, cnt, std::move(handler)); + queue_tail = new_tail; + + bool empty = (state == state_t::EMPTY); + if (empty) + state = state_t::QUEUED; lock.unlock(); - /* If it is not empty, the new heap will be started as a continuation - * of the previous one. - */ if (empty) - { - send_time = send_time_burst = timer_type::clock_type::now(); - rate_bytes = 0; - get_io_service().dispatch([this] { send_next_packet(); }); - } + get_io_service().dispatch([this] { do_next(); }); return true; } - - /** - * Block until all enqueued heaps have been sent. This function is - * thread-safe, but can be live-locked if more heaps are added while it is - * running. - */ - virtual void flush() override - { - std::unique_lock lock(queue_mutex); - while (!queue.empty()) - { - heap_empty.wait(lock); - } - } }; } // namespace send diff -Nru spead2-1.10.0/include/spead2/send_tcp.h spead2-2.1.0/include/spead2/send_tcp.h --- spead2-1.10.0/include/spead2/send_tcp.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_tcp.h 2019-11-13 08:46:10.000000000 +0000 @@ -55,14 +55,7 @@ /// Whether the underlying socket is already connected or not std::atomic connected{false}; - template - void async_send_packet(const packet &pkt, Handler &&handler) - { - if (!connected.load()) - handler(boost::asio::error::not_connected, 0); - else - boost::asio::async_write(socket, pkt.buffers, std::move(handler)); - } + void async_send_packets(); public: /// Socket send buffer size, if none is explicitly passed to the constructor @@ -95,11 +88,11 @@ const stream_config &config = stream_config(), std::size_t buffer_size = default_buffer_size, const boost::asio::ip::address &interface_address = boost::asio::ip::address()) - : stream_impl(std::move(io_service), config), + : stream_impl(std::move(io_service), config, 1), socket(detail::make_socket(get_io_service(), endpoint, buffer_size, interface_address)) { socket.async_connect(endpoint, - [this, connect_handler] (boost::system::error_code ec) + [this, connect_handler] (const boost::system::error_code &ec) { if (!ec) connected.store(true); @@ -107,12 +100,18 @@ }); } +#if BOOST_VERSION < 107000 /** * Constructor using an existing socket. The socket must be connected. + * + * @deprecated This constructor is not supported from Boost 1.70 onwards, + * and will be removed entirely in a future release. Use the constructor with + * an explicit @a io_service. */ tcp_stream( boost::asio::ip::tcp::socket &&socket, const stream_config &config = stream_config()); +#endif /** * Constructor using an existing socket. The socket must be connected. diff -Nru spead2-1.10.0/include/spead2/send_udp.h spead2-2.1.0/include/spead2/send_udp.h --- spead2-1.10.0/include/spead2/send_udp.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_udp.h 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,8 +21,17 @@ #ifndef SPEAD2_SEND_UDP_H #define SPEAD2_SEND_UDP_H +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include +#if SPEAD2_USE_SENDMMSG +# include +# include +#endif #include #include +#include #include #include @@ -38,11 +47,16 @@ boost::asio::ip::udp::socket socket; boost::asio::ip::udp::endpoint endpoint; - template - void async_send_packet(const packet &pkt, Handler &&handler) - { - socket.async_send_to(pkt.buffers, endpoint, std::move(handler)); - } + /// Implements async_send_packets, starting from @a first + void send_packets(std::size_t first); + + void async_send_packets(); + + static constexpr int batch_size = 64; +#if SPEAD2_USE_SENDMMSG + struct mmsghdr msgvec[batch_size]; + std::vector msg_iov; +#endif public: /// Socket send buffer size, if none is explicitly passed to the constructor @@ -71,11 +85,13 @@ std::size_t buffer_size = default_buffer_size, const boost::asio::ip::address &interface_address = boost::asio::ip::address()); +#if BOOST_VERSION < 107000 /** * Constructor using an existing socket. The socket must be open but * not bound. * - * @deprecated Use the overload without @a buffer_size. + * @deprecated Use the overload without @a buffer_size and with an explicit + * @a io_service. */ udp_stream( boost::asio::ip::udp::socket &&socket, @@ -84,13 +100,18 @@ std::size_t buffer_size); /** - * Constructor using an existing socket. The socket must be open and + * Constructor using an existing socket. The socket must be open * but not connected. + * + * @deprecated This constructor is not supported from Boost 1.70 onwards, + * and will be removed entirely in a future release. Use the constructor with + * an explicit @a io_service. */ udp_stream( boost::asio::ip::udp::socket &&socket, const boost::asio::ip::udp::endpoint &endpoint, const stream_config &config = stream_config()); +#endif // BOOST_VERSION < 107000 /** * Constructor using an existing socket and an explicit io_service or diff -Nru spead2-1.10.0/include/spead2/send_udp_ibv.h spead2-2.1.0/include/spead2/send_udp_ibv.h --- spead2-1.10.0/include/spead2/send_udp_ibv.h 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/include/spead2/send_udp_ibv.h 2020-02-14 08:26:39.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016 SKA South Africa +/* Copyright 2016, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -29,9 +29,8 @@ #include #include -#include #include -#include +#include #include #include #include @@ -58,7 +57,6 @@ ibv_send_wr wr{}; ibv_sge sge{}; ethernet_frame frame; - std::function handler; }; const std::size_t n_slots; @@ -76,6 +74,7 @@ ibv_mr_t mr; std::unique_ptr slots; std::vector available; + const int max_poll; static ibv_qp_t create_qp(const ibv_pd_t &pd, const ibv_cq_t &send_cq, const ibv_cq_t &recv_cq, @@ -84,47 +83,17 @@ /// Clear out the completion queue and return slots to available void reap(); - void async_send_packet(const packet &pkt, completion_handler &&handler); - /** - * Handler triggered by a completion interrupt. It would be much simpler as - * a lambda function, but this does not allow perfect forwarding of the - * handler since C++11 doesn't have generalised lambda captures. + * Try to reap completion events until there is enough space to send all + * current packets. Returns true if successful, otherwise returns false + * and schedules @ref async_send_packets to be called again later. */ - struct rerun_async_send_packet - { - private: - udp_ibv_stream *self; - const packet *pkt; - udp_ibv_stream::completion_handler handler; - - public: - rerun_async_send_packet(udp_ibv_stream *self, - const packet &pkt, - udp_ibv_stream::completion_handler &&handler); - - void operator()(boost::system::error_code ec, std::size_t bytes_transferred); - }; + bool make_space(); - /** - * Wrapper to defer invocation of the handler. Like the above, this is - * a workaround for lambdas not allowing moves. - */ - struct invoke_handler - { - private: - udp_ibv_stream::completion_handler handler; - boost::system::error_code ec; - std::size_t bytes_transferred; - public: - invoke_handler(udp_ibv_stream::completion_handler &&handler, - boost::system::error_code ec, - std::size_t bytes_transferred); - void operator()(); - }; + void async_send_packets(); public: - /// Default receive buffer size, if none is passed to the constructor + /// Default send buffer size, if none is passed to the constructor static constexpr std::size_t default_buffer_size = 512 * 1024; /// Number of times to poll in a row, if none is explicitly passed to the constructor static constexpr int default_max_poll = 10; diff -Nru spead2-1.10.0/Makefile.in spead2-2.1.0/Makefile.in --- spead2-1.10.0/Makefile.in 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/Makefile.in 2020-02-14 08:26:58.000000000 +0000 @@ -201,12 +201,11 @@ CSCOPE = cscope DIST_SUBDIRS = $(SUBDIRS) am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/spead2.pc.in \ - $(top_srcdir)/build-aux/compile \ $(top_srcdir)/build-aux/install-sh \ $(top_srcdir)/build-aux/missing \ $(top_srcdir)/include/spead2/common_features.h.in COPYING \ - COPYING.LESSER INSTALL build-aux/compile build-aux/depcomp \ - build-aux/install-sh build-aux/missing + COPYING.LESSER INSTALL TODO build-aux/compile \ + build-aux/depcomp build-aux/install-sh build-aux/missing DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) distdir = $(PACKAGE)-$(VERSION) top_distdir = $(distdir) @@ -257,9 +256,6 @@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXDEPMODE = @CXXDEPMODE@ @@ -297,19 +293,19 @@ SPEAD2_USE_EVENTFD = @SPEAD2_USE_EVENTFD@ SPEAD2_USE_IBV = @SPEAD2_USE_IBV@ SPEAD2_USE_IBV_EXP = @SPEAD2_USE_IBV_EXP@ +SPEAD2_USE_IBV_MPRQ = @SPEAD2_USE_IBV_MPRQ@ SPEAD2_USE_MOVNTDQ = @SPEAD2_USE_MOVNTDQ@ -SPEAD2_USE_NETMAP = @SPEAD2_USE_NETMAP@ SPEAD2_USE_PCAP = @SPEAD2_USE_PCAP@ SPEAD2_USE_POSIX_SEMAPHORES = @SPEAD2_USE_POSIX_SEMAPHORES@ SPEAD2_USE_PTHREAD_SETAFFINITY_NP = @SPEAD2_USE_PTHREAD_SETAFFINITY_NP@ SPEAD2_USE_RECVMMSG = @SPEAD2_USE_RECVMMSG@ +SPEAD2_USE_SENDMMSG = @SPEAD2_USE_SENDMMSG@ STRIP = @STRIP@ VERSION = @VERSION@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ diff -Nru spead2-1.10.0/MANIFEST.in spead2-2.1.0/MANIFEST.in --- spead2-1.10.0/MANIFEST.in 2018-10-11 12:50:23.000000000 +0000 +++ spead2-2.1.0/MANIFEST.in 2019-10-07 14:26:11.000000000 +0000 @@ -1,9 +1,9 @@ global-include Makefile.in global-include aclocal.m4 -global-include *asyncio.py exclude src/common_features.h exclude .gitignore include configure +include include/spead2/common_ibv_loader.h graft 3rdparty graft build-aux prune doc/_build diff -Nru spead2-1.10.0/manylinux/Dockerfile spead2-2.1.0/manylinux/Dockerfile --- spead2-1.10.0/manylinux/Dockerfile 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/manylinux/Dockerfile 2019-10-08 08:00:52.000000000 +0000 @@ -0,0 +1,25 @@ +FROM quay.io/pypa/manylinux2010_x86_64 + +RUN yum install -y wget libpcap libpcap-devel libnl python-devel libnl-devel valgrind-devel redhat-rpm-config rpm-build gcc + +RUN wget https://ufpr.dl.sourceforge.net/project/boost/boost/1.70.0/boost_1_70_0.tar.bz2 -O /tmp/boost_1_70_0.tar.bz2 +RUN tar -C /tmp -jxf /tmp/boost_1_70_0.tar.bz2 && \ + cd /tmp/boost_1_70_0 && \ + ./bootstrap.sh --prefix=/opt/boost_1_70_0 --with-libraries=program_options,system && \ + ./b2 cxxflags=-fPIC link=static install && \ + cd / && rm -rf /tmp/boost_1_70_0* + +RUN wget http://www.mellanox.com/downloads/ofed/MLNX_OFED-4.5-1.0.1.0/MLNX_OFED_SRC-4.5-1.0.1.0.tgz -O /tmp/MLNX_OFED_SRC-4.5-1.0.1.0.tgz +RUN tar -C /tmp -zxf /tmp/MLNX_OFED_SRC-4.5-1.0.1.0.tgz +RUN cd /tmp/MLNX_OFED_SRC-4.5-1.0.1.0 && \ + echo -e "libibverbs=y\nlibibverbs-devel=y\nlibrdmacm=y\nlibrdmacm-devel=y\n" > ofed.cfg && \ + ./install.pl --user-space-only -c ofed.cfg + +ENV CPATH=/opt/boost_1_70_0/include LIBRARY_PATH=/opt/boost_1_70_0/lib + +COPY requirements.txt /tmp/spead2/requirements.txt +COPY manylinux/install_requirements.sh /tmp/spead2/manylinux/install_requirements.sh +RUN /tmp/spead2/manylinux/install_requirements.sh + +COPY . /tmp/spead2 +RUN /tmp/spead2/manylinux/generate_wheels.sh diff -Nru spead2-1.10.0/manylinux/generate_wheels.sh spead2-2.1.0/manylinux/generate_wheels.sh --- spead2-1.10.0/manylinux/generate_wheels.sh 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/manylinux/generate_wheels.sh 2019-10-15 14:42:20.000000000 +0000 @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +cd /tmp/spead2 +mkdir -p /output +version="$(sed 's/.*"\(.*\)"/\1/' spead2/_version.py)" +for d in /opt/python/cp{35,36,37,38}*; do + git clean -xdf + $d/bin/pip install jinja2==2.10.1 pycparser==2.19 # For bootstrap + PATH=$d/bin:$PATH ./bootstrap.sh + echo "[build_ext]" > setup.cfg + echo "split_debug = /output" >> setup.cfg + $d/bin/pip wheel --no-deps . + auditwheel repair --plat manylinux2010_x86_64 -w /output spead2-*-`basename $d`-linux_*.whl +done +cd /output +tar -Jcvf spead2-$version-debug.tar.xz _spead2*.debug diff -Nru spead2-1.10.0/manylinux/install_requirements.sh spead2-2.1.0/manylinux/install_requirements.sh --- spead2-1.10.0/manylinux/install_requirements.sh 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/manylinux/install_requirements.sh 2019-10-07 14:23:49.000000000 +0000 @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +for d in /opt/python/cp{35,36,37}*; do + $d/bin/pip install -r /tmp/spead2/requirements.txt +done diff -Nru spead2-1.10.0/manylinux1/Dockerfile spead2-2.1.0/manylinux1/Dockerfile --- spead2-1.10.0/manylinux1/Dockerfile 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/manylinux1/Dockerfile 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -FROM quay.io/pypa/manylinux1_x86_64 - -RUN curl https://ufpr.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 > /tmp/boost_1_64_0.tar.bz2 -RUN tar -C /tmp -jxf /tmp/boost_1_64_0.tar.bz2 && \ - cd /tmp/boost_1_64_0 && \ - ./bootstrap.sh --prefix=/opt/boost_1_64_0 --with-libraries=program_options,system && \ - ./b2 cxxflags=-fPIC link=static install && \ - cd / && rm -rf /tmp/boost_1_64_0* - -ENV CPATH=/opt/boost_1_64_0/include LIBRARY_PATH=/opt/boost_1_64_0/lib - -COPY requirements.txt /tmp/spead2/requirements.txt -COPY manylinux1/install_requirements.sh /tmp/spead2/manylinux1/install_requirements.sh -RUN /tmp/spead2/manylinux1/install_requirements.sh - -COPY . /tmp/spead2 -RUN /tmp/spead2/manylinux1/generate_wheels.sh diff -Nru spead2-1.10.0/manylinux1/generate_wheels.sh spead2-2.1.0/manylinux1/generate_wheels.sh --- spead2-1.10.0/manylinux1/generate_wheels.sh 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/manylinux1/generate_wheels.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -#!/bin/bash -set -e - -cd /tmp/spead2 -mkdir -p /output -for d in /opt/python/cp{27,34,35,36,37}*; do - git clean -xdf - PATH=$d/bin:$PATH ./bootstrap.sh - $d/bin/python ./setup.py bdist_wheel -d . - auditwheel repair -w /output spead2-*-`basename $d`-linux_*.whl -done diff -Nru spead2-1.10.0/manylinux1/install_requirements.sh spead2-2.1.0/manylinux1/install_requirements.sh --- spead2-1.10.0/manylinux1/install_requirements.sh 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/manylinux1/install_requirements.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -for d in /opt/python/cp{27,34,35,36,37}*; do - $d/bin/pip install -r /tmp/spead2/requirements.txt -done diff -Nru spead2-1.10.0/mkwheels.sh spead2-2.1.0/mkwheels.sh --- spead2-1.10.0/mkwheels.sh 2018-10-12 06:44:38.000000000 +0000 +++ spead2-2.1.0/mkwheels.sh 2019-10-15 14:42:20.000000000 +0000 @@ -1,7 +1,10 @@ #!/bin/sh set -e -sudo docker build --pull -t ska-sa/spead2/manylinux1 -f manylinux1/Dockerfile . -mkdir -p wheelhouse -sudo docker run --rm -v "$PWD/wheelhouse:/wheelhouse" ska-sa/spead2/manylinux1 sh -c 'cp -v /output/*.whl /wheelhouse' -sudo chown `id -u`:`id -g` wheelhouse/*.whl +sudo docker build --pull -t ska-sa/spead2/manylinux -f manylinux/Dockerfile . +mkdir -p wheelhouse debug-symbols +sudo docker run --rm -v "$PWD/wheelhouse:/wheelhouse" -v "$PWD/debug-symbols:/debug-symbols" ska-sa/spead2/manylinux sh -c ' + cp -v /output/*.whl /wheelhouse + cp -v /output/*-debug.tar.xz /debug-symbols +' +sudo chown `id -u`:`id -g` wheelhouse/*.whl debug-symbols/*.tar.xz diff -Nru spead2-1.10.0/PKG-INFO spead2-2.1.0/PKG-INFO --- spead2-1.10.0/PKG-INFO 2018-12-11 11:46:36.000000000 +0000 +++ spead2-2.1.0/PKG-INFO 2020-02-14 08:54:39.000000000 +0000 @@ -1,19 +1,54 @@ -Metadata-Version: 1.1 +Metadata-Version: 1.2 Name: spead2 -Version: 1.10.0 +Version: 2.1.0 Summary: High-performance SPEAD implementation Home-page: https://github.com/ska-sa/spead2 Author: Bruce Merry Author-email: bmerry@ska.ac.za License: LGPLv3+ -Description: UNKNOWN +Description: spead2 + ====== + + .. image:: https://readthedocs.org/projects/spead2/badge/?version=latest + :target: https://readthedocs.org/projects/spead2/?badge=latest + :alt: Documentation Status + + .. image:: https://travis-ci.org/ska-sa/spead2.svg?branch=master + :target: https://travis-ci.org/ska-sa/spead2 + :alt: Test Status + + .. image:: https://coveralls.io/repos/github/ska-sa/spead2/badge.svg + :target: https://coveralls.io/github/ska-sa/spead2 + :alt: Coverage Status + + spead2 is an implementation of the SPEAD_ protocol, with both Python and C++ + bindings. The *2* in the name indicates that this is a new implementation of + the protocol; the protocol remains essentially the same. Compared to the + PySPEAD_ implementation, spead2: + + - is at least an order of magnitude faster when dealing with large heaps; + - correctly implements several aspects of the protocol that were implemented + incorrectly in PySPEAD (bug-compatibility is also available); + - correctly implements many corner cases on which PySPEAD would simply fail; + - cleanly supports several SPEAD flavours (e.g. 64-40 and 64-48) in one + module, with the receiver adapting to the flavour used by the sender; + - supports Python 3; + - supports asynchronous operation, using asyncio_. + + For more information, refer to the documentation on readthedocs_. + + .. _SPEAD: https://casper.berkeley.edu/wiki/SPEAD + .. _PySPEAD: https://github.com/ska-sa/PySPEAD/ + .. _asyncio: https://docs.python.org/3/library/asyncio.html + .. _readthedocs: http://spead2.readthedocs.io/en/latest/ + Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Framework :: AsyncIO Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) Classifier: Operating System :: POSIX -Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: System :: Networking +Requires-Python: >=3.5 diff -Nru spead2-1.10.0/README.rst spead2-2.1.0/README.rst --- spead2-1.10.0/README.rst 2018-12-11 10:54:54.000000000 +0000 +++ spead2-2.1.0/README.rst 2019-09-17 20:08:17.000000000 +0000 @@ -25,12 +25,11 @@ - cleanly supports several SPEAD flavours (e.g. 64-40 and 64-48) in one module, with the receiver adapting to the flavour used by the sender; - supports Python 3; -- supports asynchronous operation, using trollius_ or asyncio_. +- supports asynchronous operation, using asyncio_. For more information, refer to the documentation on readthedocs_. .. _SPEAD: https://casper.berkeley.edu/wiki/SPEAD .. _PySPEAD: https://github.com/ska-sa/PySPEAD/ -.. _trollius: http://trollius.readthedocs.io/ .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _readthedocs: http://spead2.readthedocs.io/en/latest/ diff -Nru spead2-1.10.0/.readthedocs.yml spead2-2.1.0/.readthedocs.yml --- spead2-1.10.0/.readthedocs.yml 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/.readthedocs.yml 2019-06-03 10:02:21.000000000 +0000 @@ -1,4 +1,7 @@ -requirements_file: requirements-readthedocs.txt +version: 2 python: - pip_install: true - python_version: 3 + version: 3.6 + install: + - requirements: requirements-readthedocs.txt + - method: pip + path: . diff -Nru spead2-1.10.0/RELEASE-PROCESS spead2-2.1.0/RELEASE-PROCESS --- spead2-1.10.0/RELEASE-PROCESS 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/RELEASE-PROCESS 2019-10-15 14:42:20.000000000 +0000 @@ -1,12 +1,14 @@ - pip install setuptools_git - ./bootstrap.sh -- Update version number in ChangeLog +- Update version number in doc/changelog.rst - Update version number in spead2/_version.py +- Check that .pyi stubs have been updated - Check that Travis successfully tested the release - Run ./mkwheels.sh - Run ./setup.py sdist -- Unpack the sdist and check that it installs and passes nosetests -- Install a wheel and check that it installs and passes nosetests +- Install the sdist and check that it passes nosetests +- Install a wheel and check that it passes nosetests - Tag the release, git push --tags -- Upload the sdist to PyPI with twine -- Upload the sdist and wheels to github +- Upload the sdist and wheels to PyPI with twine +- Upload the sdist and debug symbols to github +- Check that readthedocs has updated itself diff -Nru spead2-1.10.0/requirements.in spead2-2.1.0/requirements.in --- spead2-1.10.0/requirements.in 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/requirements.in 2019-10-07 14:23:49.000000000 +0000 @@ -1,9 +1,8 @@ -asynctest; python_version>='3.5' +asynctest +flake8 netifaces numpy nose nose-ignore-docstring -decorator -six -trollius; python_version<'3.7' -trollius-fixers +pycparser +jinja2 diff -Nru spead2-1.10.0/requirements-readthedocs.txt spead2-2.1.0/requirements-readthedocs.txt --- spead2-1.10.0/requirements-readthedocs.txt 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/requirements-readthedocs.txt 2019-06-03 10:02:21.000000000 +0000 @@ -1,8 +1,3 @@ -netifaces -numpy -nose -decorator -six -sphinx -breathe -trollius +sphinx==2.1.0 +breathe==4.13.0.post0 +-r requirements.txt diff -Nru spead2-1.10.0/requirements.txt spead2-2.1.0/requirements.txt --- spead2-1.10.0/requirements.txt 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/requirements.txt 2019-10-07 14:23:49.000000000 +0000 @@ -2,15 +2,18 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in +# pip-compile requirements.in # -asynctest==0.12.2 ; python_version >= "3.5" -decorator==4.0.11 -futures==3.1.1 # via trollius -netifaces==0.10.6 +asynctest==0.13.0 +entrypoints==0.3 # via flake8 +flake8==3.7.8 +jinja2==2.10.1 +markupsafe==1.1.1 # via jinja2 +mccabe==0.6.1 # via flake8 +netifaces==0.10.9 nose-ignore-docstring==0.2 nose==1.3.7 -numpy==1.14.5 -six==1.10.0 -trollius-fixers==0.1 -trollius==2.1 ; python_version < "3.7" +numpy==1.17.2 +pycodestyle==2.5.0 # via flake8 +pycparser==2.19 +pyflakes==2.1.1 # via flake8 diff -Nru spead2-1.10.0/scripts/spead2_bench.py spead2-2.1.0/scripts/spead2_bench.py --- spead2-1.10.0/scripts/spead2_bench.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/scripts/spead2_bench.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Benchmark tool to estimate the sustainable SPEAD bandwidth between two -machines, for a specific set of configurations. - -Since UDP is lossy, this is not a trivial problem. We binary search for the -speed that is just sustainable. To make the test at a specific speed more -reliable, it is repeated several times, opening a new stream each time, and -with a delay to allow processors to return to idle states. A TCP control -stream is used to synchronise the two ends. All configuration is done on -the master end. -""" - -import sys - - -if sys.version_info >= (3, 4): - from spead2.tools import bench_asyncio - bench_asyncio.main() -else: - from spead2.tools import bench_trollius - bench_trollius.main() diff -Nru spead2-1.10.0/scripts/spead2_recv.py spead2-2.1.0/scripts/spead2_recv.py --- spead2-1.10.0/scripts/spead2_recv.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/scripts/spead2_recv.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Receive SPEAD packets and log the contents. - -This is both a tool for debugging SPEAD data flows and a demonstrator for the -spead2 package. It thus has many more command-line options than are strictly -necessary, to allow multiple code-paths to be exercised. -""" - -import sys - - -if sys.version_info >= (3, 4): - from spead2.tools import recv_asyncio - recv_asyncio.main() -else: - from spead2.tools import recv_trollius - recv_trollius.main() diff -Nru spead2-1.10.0/scripts/spead2_send.py spead2-2.1.0/scripts/spead2_send.py --- spead2-1.10.0/scripts/spead2_send.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/scripts/spead2_send.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Generate and send SPEAD packets. This is mainly a benchmark application, but also -demonstrates the API.""" - -import sys - - -if sys.version_info >= (3, 4): - from spead2.tools import send_asyncio - send_asyncio.main() -else: - from spead2.tools import send_trollius - send_trollius.main() diff -Nru spead2-1.10.0/setup.py spead2-2.1.0/setup.py --- spead2-1.10.0/setup.py 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/setup.py 2019-10-15 14:42:20.000000000 +0000 @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2015, 2017 SKA South Africa +# Copyright 2015, 2017, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -18,13 +18,9 @@ from __future__ import print_function from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext -from setuptools.command.build_py import build_py import glob -import sys import os import os.path -import ctypes.util -import re import subprocess @@ -38,51 +34,41 @@ return globals_['__version__'] -# Restrict installed modules to those appropriate to the Python version -class BuildPy(build_py): - def find_package_modules(self, package, package_dir): - # distutils uses old-style classes, so no super - modules = build_py.find_package_modules(self, package, package_dir) - if sys.version_info < (3, 4): - modules = [m for m in modules if not m[1].endswith('asyncio')] - if sys.version_info < (3, 5): - modules = [m for m in modules if not m[1].endswith('py35')] - if sys.version_info >= (3, 7): - modules = [m for m in modules if not m[1].endswith('trollius')] - return modules - - class BuildExt(build_ext): user_options = build_ext.user_options + [ ('coverage', None, - "build with GCC --coverage option") + 'build with GCC --coverage option'), + ('split-debug=', None, + 'write debug symbols to a separate directory') ] boolean_options = build_ext.boolean_options + ['coverage'] def initialize_options(self): build_ext.initialize_options(self) - self.coverage = None + # setuptools bug causes these to be lost during reinitialization by + # ./setup.py develop + if not hasattr(self, 'coverage'): + self.coverage = None + if not hasattr(self, 'split_debug'): + self.split_debug = None def run(self): self.mkpath(self.build_temp) subprocess.check_call(os.path.abspath('configure'), cwd=self.build_temp) # Ugly hack to add libraries conditional on configure result - have_ibv = False have_pcap = False with open(os.path.join(self.build_temp, 'include', 'spead2', 'common_features.h')) as f: for line in f: - if line.strip() == '#define SPEAD2_USE_IBV 1': - have_ibv = True if line.strip() == '#define SPEAD2_USE_PCAP 1': have_pcap = True for extension in self.extensions: - if have_ibv: - extension.libraries.extend(['rdmacm', 'ibverbs']) if have_pcap: extension.libraries.extend(['pcap']) if self.coverage: extension.extra_compile_args.extend(['-g', '--coverage']) extension.libraries.extend(['gcov']) + if self.split_debug: + extension.extra_compile_args.extend(['-g']) extension.include_dirs.insert(0, os.path.join(self.build_temp, 'include')) # distutils uses old-style classes, so no super build_ext.run(self) @@ -95,7 +81,38 @@ pass build_ext.build_extensions(self) -# Can't actually install on readthedocs.org because Boost.Python is missing, + def build_extension(self, ext): + ext_path = self.get_ext_fullpath(ext.name) + if self.split_debug: + # If the base class decides to skip the link, we'll end up + # constructing the .debug file from the already-stripped version of + # the library, and it won't have any symbols. So force the link by + # removing the output. + try: + os.remove(ext_path) + except OSError: + pass + build_ext.build_extension(self, ext) + if self.split_debug: + os.makedirs(self.split_debug, exist_ok=True) + basename = os.path.basename(ext_path) + debug_path = os.path.join(self.split_debug, basename + '.debug') + self.spawn(['objcopy', '--only-keep-debug', '--', ext_path, debug_path]) + self.spawn(['strip', '--strip-debug', '--', ext_path]) + old_cwd = os.getcwd() + # See the documentation for --add-gnu-debuglink for why it needs to be + # run from the directory containing the file. + ext_path_abs = os.path.abspath(ext_path) + try: + os.chdir(self.split_debug) + self.spawn(['objcopy', '--add-gnu-debuglink=' + os.path.basename(debug_path), + '--', ext_path_abs]) + finally: + os.chdir(old_cwd) + self.spawn(['chmod', '-x', '--', debug_path]) + + +# Can't actually install on readthedocs.org because we can't compile, # but we need setup.py to still be successful to make the doc build work. rtd = os.environ.get('READTHEDOCS') == 'True' @@ -130,40 +147,47 @@ else: extensions = [] +with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme_file: + readme = readme_file.read() + setup( author='Bruce Merry', author_email='bmerry@ska.ac.za', name='spead2', version=find_version(), description='High-performance SPEAD implementation', + long_description=readme, url='https://github.com/ska-sa/spead2', license='LGPLv3+', - classifiers = [ + classifiers=[ 'Development Status :: 5 - Production/Stable', 'Framework :: AsyncIO', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 'Operating System :: POSIX', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries', 'Topic :: System :: Networking'], ext_package='spead2', ext_modules=extensions, - cmdclass={'build_ext': BuildExt, 'build_py': BuildPy}, + cmdclass={'build_ext': BuildExt}, install_requires=[ - 'numpy>=1.9.2', - 'six', - 'trollius; python_version<"3.4"' + 'numpy>=1.9.2' ], tests_require=[ 'netifaces', 'nose', - 'decorator', - 'trollius; python_version<"3.7"', - 'asynctest; python_version>="3.5"' + 'asynctest' ], + python_requires='>=3.5', test_suite='nose.collector', packages=find_packages(), - scripts=glob.glob('scripts/*.py') + package_data={'': ['py.typed', '*.pyi']}, + entry_points={ + 'console_scripts': [ + 'spead2_send.py = spead2.tools.send_asyncio:main', + 'spead2_recv.py = spead2.tools.recv_asyncio:main', + 'spead2_bench.py = spead2.tools.bench_asyncio:main' + ] + } ) diff -Nru spead2-1.10.0/spead2/__init__.py spead2-2.1.0/spead2/__init__.py --- spead2-1.10.0/spead2/__init__.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/__init__.py 2019-10-07 14:26:11.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -13,9 +13,15 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import numbers as _numbers +import logging +import struct + +import numpy as _np + import spead2._spead2 -from spead2._spead2 import ( - Flavour, ThreadPool, Stopped, Empty, Stopped, +from spead2._spead2 import ( # noqa: F401 + Flavour, ThreadPool, Stopped, Empty, MemoryAllocator, MmapAllocator, MemoryPool, InprocQueue, BUG_COMPAT_DESCRIPTOR_WIDTHS, BUG_COMPAT_SHAPE_BIT_1, @@ -40,11 +46,11 @@ CTRL_DESCRIPTOR_UPDATE, MEMCPY_STD, MEMCPY_NONTEMPORAL) -import numbers as _numbers -import numpy as _np -import logging -import six -from spead2._version import __version__ +try: + from spead2._spead2 import IbvContext # noqa: F401 +except ImportError: + pass +from spead2._version import __version__ # noqa: F401 _logger = logging.getLogger(__name__) @@ -55,16 +61,6 @@ _FASTPATH_NUMPY = 2 -if six.PY2: - def _bytes_to_str_ascii(b): - b.decode('ascii') # Just to check validity, throw away unicode object - return b -else: - # Python 3 - def _bytes_to_str_ascii(b): - return b.decode('ascii') - - def _shape_elements(shape): elements = 1 for dimension in shape: @@ -91,16 +87,18 @@ return out -class Descriptor(object): +class Descriptor: """Metadata for a SPEAD item. - There are a number of restrictions in the way the parameters combine, - which will cause `ValueError` to be raised if violated: + There are a number of restrictions on the parameters, which will cause + `ValueError` to be raised if violated: - At most one element of `shape` can be `None`. - Exactly one of `dtype` and `format` must be non-`None`. - If `dtype` is specified, `shape` cannot have any unknown dimensions. - If `format` is specified, `order` must be 'C' + - If `dtype` is specified, it cannot contain objects, and must have + positive size. Parameters ---------- @@ -129,9 +127,11 @@ if dtype is not None: dtype = _np.dtype(dtype) if dtype.hasobject: - raise TypeError('Cannot use dtype that has reference-counted objects') + raise ValueError('Cannot use dtype that has reference-counted objects') if format is not None: raise ValueError('Only one of dtype and format can be specified') + if dtype.itemsize == 0: + raise ValueError('Cannot use zero-sized dtype') if unknowns > 0: raise ValueError('Cannot have unknown dimensions when using numpy descriptor') self._internal_dtype = dtype @@ -168,7 +168,7 @@ d = _np.lib.utils.safe_eval(header) except SyntaxError as e: msg = "Cannot parse descriptor: %r\nException: %r" - raise ValueError(msg % (header, e)) + raise ValueError(msg % (header, e)) from e if not isinstance(d, dict): msg = "Descriptor is not a dictionary: %r" raise ValueError(msg % d) @@ -189,15 +189,15 @@ dtype = _np.dtype(d['descr']) except TypeError as e: msg = "descr is not a valid dtype descriptor: %r" - raise ValueError(msg % (d['descr'],)) + raise ValueError(msg % (d['descr'],)) from e order = 'F' if d['fortran_order'] else 'C' return d['shape'], order, dtype @classmethod def _make_numpy_header(self, shape, dtype, order): return "{{'descr': {!r}, 'fortran_order': {!r}, 'shape': {!r}}}".format( - _np.lib.format.dtype_to_descr(dtype), order == 'F', - tuple(shape)) + _np.lib.format.dtype_to_descr(dtype), order == 'F', + tuple(shape)) @classmethod def _parse_format(cls, fmt): @@ -213,8 +213,11 @@ if not fmt: raise ValueError('empty format') for code, length in fmt: - if length == 0: - raise ValueError('zero-length field (bug_compat mismatch?)') + if length <= 0: + if length == 0: + raise ValueError('zero-length field (bug_compat mismatch?)') + else: + raise ValueError('negative-length field') if ((code in ('u', 'i') and length in (8, 16, 32, 64)) or (code == 'f' and length in (32, 64))): fields.append('>' + code + str(length // 8)) @@ -251,7 +254,7 @@ is best not to send them at all. """ return not self.is_variable_size() and ( - self.dtype is not None or self.itemsize_bits % 8 == 0) + self.dtype is not None or self.itemsize_bits % 8 == 0) def dynamic_shape(self, max_elements): """Determine the dynamic shape, given incoming data that is big enough @@ -290,9 +293,8 @@ dtype = None format = None if raw_descriptor.numpy_header: - header = _bytes_to_str_ascii(raw_descriptor.numpy_header) - shape, order, dtype = \ - cls._parse_numpy_header(header) + header = raw_descriptor.numpy_header.decode('ascii') + shape, order, dtype = cls._parse_numpy_header(header) if flavour.bug_compat & BUG_COMPAT_SWAP_ENDIAN: dtype = dtype.newbyteorder() else: @@ -300,10 +302,10 @@ order = 'C' format = raw_descriptor.format return cls( - raw_descriptor.id, - _bytes_to_str_ascii(raw_descriptor.name), - _bytes_to_str_ascii(raw_descriptor.description), - shape, dtype, order, format) + raw_descriptor.id, + raw_descriptor.name.decode('ascii'), + raw_descriptor.description.decode('ascii'), + shape, dtype, order, format) def to_raw(self, flavour): raw = spead2._spead2.RawDescriptor() @@ -316,7 +318,8 @@ dtype = self.dtype.newbyteorder() else: dtype = self.dtype - raw.numpy_header = self._make_numpy_header(self.shape, dtype, self.order).encode('ascii') + raw.numpy_header = self._make_numpy_header( + self.shape, dtype, self.order).encode('ascii') else: raw.format = self.format return raw @@ -333,7 +336,7 @@ def __init__(self, *args, **kw): value = kw.pop('value', None) - super(Item, self).__init__(*args, **kw) + super().__init__(*args, **kw) self._value = value self.version = 1 #: Version number @@ -428,16 +431,16 @@ elif code == 'b': field = bool(raw) elif code == 'c': - field = six.int2byte(raw) + field = struct.pack('B', raw) elif code == 'f': if length == 32: field = _np.uint32(raw).view(_np.float32) elif length == 64: field = _np.uint64(raw).view(_np.float64) else: - raise ValueError('unhandled float length {0}'.format((code, length))) + raise ValueError('unhandled float length {}'.format((code, length))) else: - raise ValueError('unhandled format {0}'.format((code, length))) + raise ValueError('unhandled format {}'.format((code, length))) fields.append(field) if len(fields) == 1: ans = fields[0] @@ -476,12 +479,12 @@ elif length == 64: raw = _np.float64(field).view(_np.uint64) else: - raise ValueError('unhandled float length {0}'.format((code, length))) + raise ValueError('unhandled float length {}'.format((code, length))) else: - raise ValueError('unhandled format {0}'.format((code, length))) + raise ValueError('unhandled format {}'.format((code, length))) gen.send((raw, length)) - def set_from_raw(self, raw_item): + def set_from_raw(self, raw_item, new_order='='): raw_value = _np.array(raw_item, _np.uint8, copy=False) if self._fastpath == _FASTPATH_NUMPY: max_elements = raw_value.shape[0] // self._internal_dtype.itemsize @@ -498,8 +501,9 @@ else: array1d = raw_value[:size_bytes] array1d = array1d.view(dtype=self._internal_dtype) - # Force to native endian - array1d = array1d.astype(self._internal_dtype.newbyteorder('='), casting='equiv', copy=False) + # Force the byte order if requested + array1d = array1d.astype(self._internal_dtype.newbyteorder(new_order), + casting='equiv', copy=False) value = _np.reshape(array1d, shape, self.order) elif (self._fastpath == _FASTPATH_IMMEDIATE and raw_item.is_immediate and @@ -532,7 +536,7 @@ value = value[()] elif len(self.shape) == 1 and self.format == [('c', 8)]: # Convert array of characters to a string - value = _bytes_to_str_ascii(b''.join(value)) + value = b''.join(value).decode('ascii') self.value = value def _num_elements(self): @@ -573,12 +577,11 @@ value = self.value if value is None: raise ValueError('Cannot send a value of None') - if (isinstance(value, (six.binary_type, six.text_type)) and - len(self.shape) == 1): + if isinstance(value, (bytes, str)) and len(self.shape) == 1: # This is complicated by Python 3 not providing a simple way to # turn a bytes object into a list of one-byte objects, the way # list(str) does. - value = [self.value[i : i+1] for i in range(len(self.value))] + value = [self.value[i : i + 1] for i in range(len(self.value))] value = _np.array(value, dtype=self._internal_dtype, order=self.order, copy=False) if not self.compatible_shape(value.shape): raise ValueError('Value has shape {}, expected {}'.format(value.shape, self.shape)) @@ -612,7 +615,7 @@ return value -class ItemGroup(object): +class ItemGroup: """ Items are collected into sets called *item groups*, which can be indexed by either item ID or item name. @@ -649,12 +652,12 @@ # Check if this is just the same thing if (old is not None and - old.name == item.name and - old.description == item.description and - old.shape == item.shape and - old.dtype == item.dtype and - old.order == item.order and - old.format == item.format): + old.name == item.name and + old.description == item.description and + old.shape == item.shape and + old.dtype == item.dtype and + old.order == item.order and + old.format == item.format): # Descriptor is the same, so just transfer the value. If the value # is None, then we've only been given a descriptor to add. if item.value is not None: @@ -734,13 +737,18 @@ """Number of items""" return len(self._by_name) - def update(self, heap): + def update(self, heap, new_order='='): """Update the item descriptors and items from an incoming heap. Parameters ---------- heap : :class:`spead2.recv.Heap` Incoming heap + new_order : str + Byte ordering to coerce new byte arrays into. The default is to + force arrays to native byte order. Use ``'|'`` to keep whatever + byte order was in the heap. See :meth:`np.dtype.newbyteorder` for + the full list of options. Returns ------- @@ -759,7 +767,7 @@ except KeyError: _logger.warning('Item with ID %#x received but there is no descriptor', raw_item.id) else: - item.set_from_raw(raw_item) + item.set_from_raw(raw_item, new_order=new_order) item.version += 1 updated_items[item.name] = item return updated_items diff -Nru spead2-1.10.0/spead2/__init__.pyi spead2-2.1.0/spead2/__init__.pyi --- spead2-1.10.0/spead2/__init__.pyi 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/__init__.pyi 2019-11-12 06:29:43.000000000 +0000 @@ -0,0 +1,194 @@ +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from typing import (List, Sequence, Optional, Tuple, Any, Union, + Dict, KeysView, ValuesView, Text, overload) + +import numpy as np +import spead2.recv + + +# Not the same as typing.AnyStr, which is a TypeVar +_PybindStr = Union[Text, bytes] + +__version__: str + +BUG_COMPAT_DESCRIPTOR_WIDTHS: int +BUG_COMPAT_SHAPE_BIT_1: int +BUG_COMPAT_SWAP_ENDIAN: int +BUG_COMPAT_PYSPEAD_0_5_2: int + +NULL_ID: int +HEAP_CNT_ID: int +HEAP_LENGTH_ID: int +PAYLOAD_OFFSET_ID: int +PAYLOAD_LENGTH_ID: int +DESCRIPTOR_ID: int +STREAM_CTRL_ID: int + +DESCRIPTOR_NAME_ID: int +DESCRIPTOR_DESCRIPTION_ID: int +DESCRIPTOR_SHAPE_ID: int +DESCRIPTOR_FORMAT_ID: int +DESCTIPTOR_DTYPE_ID: int + +CTRL_STREAM_START: int +CTRL_DESCRIPTOR_REISSUE: int +CTRL_STREAM_STOP: int +CTRL_DESCRIPTOR_UPDATE: int + +MEMCPY_STD: int +MEMCPY_NONTEMPORAL: int + +class Stopped(RuntimeError): + pass + +class Empty(RuntimeError): + pass + +class Flavour(object): + @overload + def __init__(self, version: int, item_pointer_bits : int, + heap_address_bits: int, bug_compat: int = ...) -> None: ... + @overload + def __init__(self) -> None: ... + def __eq__(self, o: object) -> bool: ... + def __ne__(self, o: object) -> bool: ... + @property + def version(self) -> int: ... + @property + def item_pointer_bits(self) -> int: ... + @property + def heap_address_bits(self) -> int: ... + @property + def bug_compat(self) -> int: ... + +class ThreadPool(object): + @overload + def __init__(self, threads: int = ...) -> None: ... + @overload + def __init__(self, threads: int, affinity: List[int]) -> None: ... + +class MemoryAllocator(object): + def __init__(self) -> None: ... + +class MmapAllocator(MemoryAllocator): + def __init__(self, flags: int = ...) -> None: ... + +class MemoryPool(MemoryAllocator): + @overload + def __init__(self, lower: int, upper: int, max_free: int, initial: int, + allocator: Optional[MemoryAllocator] = None) -> None: ... + @overload + def __init__(self, thread_pool: ThreadPool, lower: int, upper: int, max_free: int, initial: int, + low_water: int, allocator: MemoryAllocator) -> None: ... + @property + def warn_on_empty(self) -> bool: ... + @warn_on_empty.setter + def warn_on_empty(self, value: bool) -> None: ... + +class InprocQueue(object): + def __init__(self) -> None: ... + def stop(self) -> None: ... + +class RawDescriptor(object): + @property + def id(self) -> int: ... + @id.setter + def id(self, value: int) -> None: ... + + @property + def name(self) -> bytes: ... + @name.setter + def name(self, value: bytes) -> None: ... + + @property + def description(self) -> bytes: ... + @description.setter + def description(self, value: bytes) -> None: ... + + @property + def shape(self) -> List[Optional[int]]: ... + @shape.setter + def shape(self, value: Sequence[Optional[int]]) -> None: ... + + @property + def format(self) -> List[Tuple[_PybindStr, int]]: ... + @format.setter + def format(self, value: List[Tuple[_PybindStr, int]]) -> None: ... + + @property + def numpy_header(self) -> bytes: ... + @numpy_header.setter + def numpy_header(self, value: bytes) -> None: ... + +class IbvContext(object): + def __init__(self, interface_address: _PybindStr) -> None: ... + def reset(self) -> None: ... + +def parse_range_list(ranges: str) -> List[int]: ... + +class Descriptor(object): + id: int + name: str + description: str + shape: Sequence[Optional[int]] + dtype: Optional[np.dtype] + order: str + format: Optional[List[Tuple[str, int]]] + + def __init__(self, id: int, name: str, description: str, + shape: Sequence[Optional[int]], dtype: Optional[np.dtype] = None, + order: str = ..., format: Optional[List[Tuple[str, int]]] = None) -> None: ... + @property + def itemsize_bits(self) -> int: ... + @property + def is_variable_size(self) -> bool: ... + @property + def allow_immediate(self) -> bool: ... + def dynamic_shape(self, max_elements: int) -> Sequence[int]: ... + def compatible_shape(self, shape: Sequence[int]) -> bool: ... + @classmethod + def from_raw(cls, raw_descriptor: RawDescriptor, flavour: Flavour): ... + def to_raw(self, flavour: Flavour) -> RawDescriptor: ... + +class Item(Descriptor): + version: int + + def __init__(self, id: int, name: str, description: str, + shape: Sequence[Optional[int]], dtype: Optional[np.dtype] = None, + order: str = ..., format: Optional[List[Tuple[str, int]]] = None, + value: Any = None) -> None: ... + @property + def value(self) -> Any: ... + @value.setter + def value(self, new_value: Any) -> None: ... + def set_from_raw(self, raw_item: spead2.recv.RawItem, new_order: str = ...) -> None: ... + # typing has no buffer protocol ABC (https://bugs.python.org/issue27501) + def to_buffer(self) -> Any: ... + +class ItemGroup(object): + def __init__(self) -> None: ... + def add_item(self, id: Optional[int], name: str, description: str, + shape: Sequence[Optional[int]], dtype: Optional[np.dtype] = None, + order: str = 'C', format: Optional[List[Tuple[str, int]]] = None, + value: Any = None) -> Item: ... + def __getitem__(self, key: Union[int, str]) -> Item: ... + def __contains__(self, key: Union[int, str]) -> bool: ... + def keys(self) -> KeysView[str]: ... + def ids(self) -> KeysView[int]: ... + def values(self) -> ValuesView[Item]: ... + def __len__(self) -> int: ... + def update(self, heap: spead2.recv.Heap, new_order: str = ...) -> Dict[str, Item]: ... diff -Nru spead2-1.10.0/spead2/py.typed spead2-2.1.0/spead2/py.typed --- spead2-1.10.0/spead2/py.typed 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/py.typed 2019-01-07 13:39:15.000000000 +0000 @@ -0,0 +1 @@ +partial diff -Nru spead2-1.10.0/spead2/recv/asyncio.py spead2-2.1.0/spead2/recv/asyncio.py --- spead2-1.10.0/spead2/recv/asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/recv/asyncio.py 2019-10-08 08:03:04.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/recv/asyncio.py. Do not edit. - -# Copyright 2015, 2018 SKA South Africa +# Copyright 2015, 2018-2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -18,27 +16,22 @@ """ Integration between spead2.recv and asyncio """ -from __future__ import absolute_import import collections import functools import sys - import asyncio - import spead2.recv # Decorator from official Python documentation to have compatibility with # both 3.5.0/1 (which expected __aiter__ to return an awaitable) and 3.5.2+ -# (which expects it to return an async iterator), and modified to use legal -# Python 2 syntax. +# (which expects it to return an async iterator). if sys.version_info < (3, 5, 2): def _aiter_compat(func): @functools.wraps(func) - @asyncio.coroutine - def wrapper(self): - return (func(self)) + async def wrapper(self): + return func(self) return wrapper else: def _aiter_compat(func): @@ -68,7 +61,7 @@ self._loop = kwargs.pop('loop', None) if self._loop is None: self._loop = asyncio.get_event_loop() - super(Stream, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._waiters = collections.deque() self._listening = False @@ -111,9 +104,10 @@ self = None waiter = None - @asyncio.coroutine - def get(self, loop=None): + async def get(self, loop=None): """Coroutine that waits for a heap to become available and returns it.""" + if loop is None: + loop = self._loop self._clear_done_waiters() if not self._waiters: # If something is available directly, we can avoid going back to @@ -125,31 +119,27 @@ else: # Give the event loop a chance to run. This ensures that a # heap-processing loop cannot live-lock the event loop. - yield - return (heap) + await asyncio.sleep(0, loop=loop) + return heap - if loop is None: - loop = self._loop waiter = asyncio.Future(loop=loop) self._waiters.append(waiter) self._start_listening() - heap = (yield from(waiter)).pop() - return (heap) + try: + return (await waiter).pop() + finally: + # Prevent cyclic references when an exception is thrown + waiter = None + self = None - # Asynchronous iterator support for Python 3.5+. It's not supported with - # asyncio, but after passing through asyncio2asyncio it becomes useful. @_aiter_compat def __aiter__(self): return self - @asyncio.coroutine - def __anext__(self): + async def __anext__(self): try: - heap = yield from(self.get()) + heap = await self.get() except spead2.Stopped: raise StopAsyncIteration else: - return (heap) -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') + return heap diff -Nru spead2-1.10.0/spead2/recv/asyncio.pyi spead2-2.1.0/spead2/recv/asyncio.pyi --- spead2-1.10.0/spead2/recv/asyncio.pyi 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/recv/asyncio.pyi 2019-01-07 13:19:31.000000000 +0000 @@ -0,0 +1,29 @@ +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import asyncio +from typing import AsyncIterator, Optional + +import spead2 +import spead2.recv + +class Stream(spead2.recv._Stream): + def __init__(self, thread_pool: spead2.ThreadPool, bug_compat: int = ..., + max_heaps: int = ..., ring_heaps: int = ..., + contiguous_only: bool = ..., + incomplete_keep_payload_ranges: bool = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + async def get(self, loop: Optional[asyncio.AbstractEventLoop] = None) -> spead2.recv.Heap: ... + def __aiter__(self) -> AsyncIterator[spead2.recv.Heap]: ... diff -Nru spead2-1.10.0/spead2/recv/__init__.py spead2-2.1.0/spead2/recv/__init__.py --- spead2-1.10.0/spead2/recv/__init__.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/recv/__init__.py 2019-10-07 14:23:49.000000000 +0000 @@ -41,9 +41,7 @@ the shape is zero. - The `u`, `i`, `c` and `b` types may also be used with other sizes, but it will invoke a slow conversion process and is not recommended for large - arrays. The valid range of the `c` conversion depends on the Python - version: for Python 2 it must be 0 to 255, for Python 3 it is interpreted - as a Unicode code point. + arrays. For `c`, the value is interpreted as a Unicode code point. Two cases are treated specially: @@ -56,4 +54,4 @@ bytes, in the order they appeared in the original packet. """ -from spead2._spead2.recv import Stream, Heap, IncompleteHeap +from spead2._spead2.recv import Stream, Heap, IncompleteHeap # noqa: F401 diff -Nru spead2-1.10.0/spead2/recv/__init__.pyi spead2-2.1.0/spead2/recv/__init__.pyi --- spead2-1.10.0/spead2/recv/__init__.pyi 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/recv/__init__.pyi 2019-03-06 09:53:45.000000000 +0000 @@ -0,0 +1,135 @@ +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from typing import Iterator, Any, List, Tuple, Sequence, Union, Text, Optional, overload +import socket + +import spead2 +from spead2 import _PybindStr + +class RawItem(object): + @property + def id(self) -> int: ... + @property + def is_immediate(self) -> bool: ... + @property + def immediate_value(self) -> int: ... + +class _HeapBase(object): + @property + def cnt(self) -> int: ... + @property + def flavour(self) -> spead2.Flavour: ... + def get_items(self) -> List[RawItem]: ... + def is_start_of_stream(self) -> bool: ... + def is_end_of_stream(self) -> bool: ... + +class Heap(_HeapBase): + def get_descriptors(self) -> List[spead2.RawDescriptor]: ... + +class IncompleteHeap(_HeapBase): + @property + def heap_length(self) -> int: ... + @property + def received_length(self) -> int: ... + @property + def payload_ranges(self) -> List[Tuple[int, int]]: ... + +class StreamStats(object): + heaps: int + incomplete_heaps_evicted: int + incomplete_Heaps_flushed: int + packets: int + batches: int + worker_blocked: int + max_batch: int + single_packet_heaps: int + search_dist: int + def __add__(self, other: StreamStats) -> StreamStats: ... + def __iadd__(self, other: StreamStats) -> None: ... + +class Ringbuffer(object): + def size(self) -> int: ... + def capacity(self) -> int: ... + +# We make a dummy _Stream base class because mypy objects to the async stream +# type overloading get as a coroutine. +class _Stream(object): + DEFAULT_UDP_IBV_MAX_SIZE: int = ... + DEFAULT_UDP_IBV_BUFFER_SIZE: int = ... + DEFAULT_UDP_IBV_MAX_POLL: int = ... + DEFAULT_MAX_HEAPS: int = ... + DEFAULT_RING_HEAPS: int = ... + DEFAULT_UDP_MAX_SIZE: int = ... + DEFAULT_UDP_BUFFER_SIZE: int = ... + DEFAULT_TCP_MAX_SIZE: int = ... + DEFAULT_TCP_BUFFER_SIZE: int = ... + + def __init__(self, thread_pool: spead2.ThreadPool, bug_compat: int = ..., + max_heaps: int = ..., ring_heaps: int = ..., + contiguous_only: bool = ..., + incomplete_keep_payload_ranges: bool = ...) -> None: ... + def __iter__(self) -> Iterator[Heap]: ... + def get_nowait(self) -> Heap: ... + def set_memory_allocator(self, allocator: spead2.MemoryAllocator) -> None: ... + def set_memory_pool(self, pool: spead2.MemoryPool) -> None: ... + def set_memcpy(self, id: int) -> None: ... + @property + def stop_on_stop_item(self) -> bool: ... + @stop_on_stop_item.setter + def stop_on_stop_item(self, value: bool) -> None: ... + @property + def allow_unsized_heaps(self) -> bool: ... + @allow_unsized_heaps.setter + def allow_unsized_heaps(self, value: bool) -> None: ... + def add_buffer_reader(self, buffer: Any) -> None: ... + @overload + def add_udp_reader(self, port: int, max_size: int = ..., buffer_size: int = ..., + bind_hostname: _PybindStr = ..., socket: Optional[socket.socket] = None) -> None: ... + @overload + def add_udp_reader(self, socket: socket.socket, max_size: int = ...) -> None: ... + @overload + def add_udp_reader(self, multicast_group: _PybindStr, port: int, max_size: int = ..., + buffer_size: int = ..., interface_address: _PybindStr = ...) -> None: ... + @overload + def add_udp_reader(self, multicast_group: _PybindStr, port: int, max_size: int = ..., + buffer_size: int = ..., interface_index: int = ...) -> None: ... + @overload + def add_tcp_reader(self, port: int, max_size: int = ..., buffer_size: int = ..., + bind_hostname: _PybindStr = ...) -> None: ... + @overload + def add_tcp_reader(self, acceptor: socket.socket, max_size: int = ...) -> None: ... + @overload + def add_udp_ibv_reader(self, multicast_group: _PybindStr, port: int, + interface_address: _PybindStr, + max_size: int = ..., buffer_size: int = ..., + comp_vector: int = ..., max_poll: int = ...) -> None: ... + @overload + def add_udp_ibv_reader(self, endpoints: Sequence[Tuple[_PybindStr, int]], + interface_address: _PybindStr, + max_size: int = ..., buffer_size: int = ..., + comp_vector: int = ..., max_poll: int = ...) -> None: ... + def add_udp_pcap_file_reader(self, filename: _PybindStr) -> None: ... + def add_inproc_reader(self, queue: spead2.InprocQueue) -> None: ... + def stop(self) -> None: ... + @property + def fd(self) -> int: ... + @property + def stats(self) -> StreamStats: ... + @property + def ringbuffer(self) -> Ringbuffer: ... + +class Stream(_Stream): + def get(self) -> Heap: ... diff -Nru spead2-1.10.0/spead2/recv/trollius.py spead2-2.1.0/spead2/recv/trollius.py --- spead2-1.10.0/spead2/recv/trollius.py 2018-08-31 07:20:29.000000000 +0000 +++ spead2-2.1.0/spead2/recv/trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,150 +0,0 @@ -# Copyright 2015, 2018 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -""" -Integration between spead2.recv and trollius -""" -from __future__ import absolute_import -import collections -import functools -import sys - -import trollius -from trollius import From, Return - -import spead2.recv - - -# Decorator from official Python documentation to have compatibility with -# both 3.5.0/1 (which expected __aiter__ to return an awaitable) and 3.5.2+ -# (which expects it to return an async iterator), and modified to use legal -# Python 2 syntax. -if sys.version_info < (3, 5, 2): - def _aiter_compat(func): - @functools.wraps(func) - @trollius.coroutine - def wrapper(self): - raise Return(func(self)) - return wrapper -else: - def _aiter_compat(func): - return func - - -class Stream(spead2.recv.Stream): - """Stream where `get` is a coroutine that yields the next heap. - - Internally, it maintains a queue of waiters, each represented by a future. - When a heap becomes available, it is passed to the first waiter. We use - a callback on a file descriptor being readable, which happens when there - might be data available. The callback is enabled when we have at least one - waiter, otherwise disabled. - - The futures store a singleton list containing the heap rather than the heap - itself. This allows the reference to the heap to be explicitly cleared so - that the heap can be garbage collected sooner. - - Parameters - ---------- - loop : event loop, optional - Default event loop - """ - - def __init__(self, *args, **kwargs): - self._loop = kwargs.pop('loop', None) - if self._loop is None: - self._loop = trollius.get_event_loop() - super(Stream, self).__init__(*args, **kwargs) - self._waiters = collections.deque() - self._listening = False - - def _start_listening(self): - if not self._listening: - self._loop.add_reader(self.fd, self._ready_callback) - self._listening = True - - def _stop_listening(self): - if self._listening: - self._loop.remove_reader(self.fd) - self._listening = False - - def _clear_done_waiters(self): - """Remove waiters that are done (should only happen if they are cancelled)""" - while self._waiters and self._waiters[0].done(): - self._waiters.popleft() - if not self._waiters: - self._stop_listening() - - def _ready_callback(self): - self._clear_done_waiters() - if self._waiters: - try: - heap = self.get_nowait() - except spead2.Empty: - # Shouldn't happen, but poll may have been woken spuriously - pass - except spead2.Stopped as e: - for waiter in self._waiters: - waiter.set_exception(e) - self._waiters = [] - self._stop_listening() - else: - waiter = self._waiters.popleft() - waiter.set_result([heap]) - if not self._waiters: - self._stop_listening() - # Break cyclic references if spead2.Stopped is raised - self = None - waiter = None - - @trollius.coroutine - def get(self, loop=None): - """Coroutine that waits for a heap to become available and returns it.""" - self._clear_done_waiters() - if not self._waiters: - # If something is available directly, we can avoid going back to - # the scheduler - try: - heap = self.get_nowait() - except spead2.Empty: - pass - else: - # Give the event loop a chance to run. This ensures that a - # heap-processing loop cannot live-lock the event loop. - yield - raise Return(heap) - - if loop is None: - loop = self._loop - waiter = trollius.Future(loop=loop) - self._waiters.append(waiter) - self._start_listening() - heap = (yield From(waiter)).pop() - raise Return(heap) - - # Asynchronous iterator support for Python 3.5+. It's not supported with - # trollius, but after passing through trollius2asyncio it becomes useful. - @_aiter_compat - def __aiter__(self): - return self - - @trollius.coroutine - def __anext__(self): - try: - heap = yield From(self.get()) - except spead2.Stopped: - raise StopAsyncIteration - else: - raise Return(heap) diff -Nru spead2-1.10.0/spead2/send/asyncio.py spead2-2.1.0/spead2/send/asyncio.py --- spead2-1.10.0/spead2/send/asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/send/asyncio.py 2019-11-13 08:46:10.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/send/asyncio.py. Do not edit. - -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -19,9 +17,10 @@ Integration between spead2.send and asyncio """ from __future__ import absolute_import + import asyncio -import spead2.send + from spead2._spead2.send import UdpStreamAsyncio as _UdpStreamAsyncio from spead2._spead2.send import TcpStreamAsyncio as _TcpStreamAsyncio from spead2._spead2.send import InprocStreamAsyncio as _InprocStreamAsyncio @@ -31,7 +30,7 @@ class Wrapped(base_class): def __init__(self, *args, **kwargs): self._loop = kwargs.pop('loop', None) - super(Wrapped, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if self._loop is None: self._loop = asyncio.get_event_loop() self._active = 0 @@ -65,7 +64,7 @@ if self._active == 0: self._loop.remove_reader(self.fd) self._last_queued_future = None # Purely to free the memory - queued = super(Wrapped, self).async_send_heap(heap, callback, cnt) + queued = super().async_send_heap(heap, callback, cnt) if self._active == 0: self._loop.add_reader(self.fd, self.process_callbacks) self._active += 1 @@ -73,14 +72,13 @@ self._last_queued_future = future return future - @asyncio.coroutine - def async_flush(self): + async def async_flush(self): """Asynchronously wait for all enqueued heaps to be sent. Note that this only waits for heaps passed to :meth:`async_send_heap` prior to this call, not ones added while waiting.""" future = self._last_queued_future if future is not None: - yield from(asyncio.wait([future])) + await asyncio.wait([future]) Wrapped.__name__ = name return Wrapped @@ -109,6 +107,8 @@ """ _TcpStreamBase = _wrap_class('TcpStream', _TcpStreamAsyncio) + + class TcpStream(_TcpStreamBase): """SPEAD over TCP with asynchronous connect and sends. @@ -127,8 +127,7 @@ """ @classmethod - @asyncio.coroutine - def connect(cls, *args, **kwargs): + async def connect(cls, *args, **kwargs): """Open a connection. The arguments are the same as for the constructor of @@ -138,6 +137,7 @@ if loop is None: loop = asyncio.get_event_loop() future = asyncio.Future(loop=loop) + def callback(arg): if not future.done(): if isinstance(arg, Exception): @@ -146,8 +146,8 @@ loop.call_soon_threadsafe(future.set_result, arg) stream = cls(callback, *args, **kwargs) - yield from(future) - return (stream) + await future + return stream InprocStream = _wrap_class('InprocStream', _InprocStreamAsyncio) @@ -216,6 +216,3 @@ except ImportError: pass -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') diff -Nru spead2-1.10.0/spead2/send/asyncio.pyi spead2-2.1.0/spead2/send/asyncio.pyi --- spead2-1.10.0/spead2/send/asyncio.pyi 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/send/asyncio.pyi 2019-11-13 08:46:10.000000000 +0000 @@ -0,0 +1,87 @@ +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import asyncio +import socket +from typing import Optional, overload + +import spead2 +import spead2.send + +from spead2 import _PybindStr + +class _AsyncStream(spead2.send._Stream): + @property + def fd(self) -> int: ... + def flush(self) -> None: ... + def async_send_heap(self, heap: spead2.send.Heap, cnt: int = ..., + loop: Optional[asyncio.AbstractEventLoop] = None) -> asyncio.Future[int]: ... + async def async_flush(self) -> None: ... + +class UdpStream(spead2.send._UdpStream, _AsyncStream): + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig, + buffer_size: int, socket: socket.socket, + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig = ..., + buffer_size: int = ..., interface_address: _PybindStr = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig, + ttl: int, + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig, + ttl: int, interface_address: _PybindStr, + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig, + ttl: int, interface_index: int, + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + socket: socket.socket, hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + + +class TcpStream(spead2.send._TcpStream, _AsyncStream): + def __init__(self, thread_pool: spead2.ThreadPool, socket: socket.socket, + config: spead2.send.StreamConfig = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + @classmethod + async def connect(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: spead2.send.StreamConfig = ..., + buffer_size: int = ..., interface_address: _PybindStr = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + + +class InprocStream(spead2.send._InprocStream, _AsyncStream): + def __init__(self, thread_pool: spead2.ThreadPool, queue: spead2.InprocQueue, + config: spead2.send.StreamConfig = ..., + *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: ... + diff -Nru spead2-1.10.0/spead2/send/__init__.py spead2-2.1.0/spead2/send/__init__.py --- spead2-1.10.0/spead2/send/__init__.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/send/__init__.py 2019-10-08 08:03:04.000000000 +0000 @@ -17,23 +17,25 @@ from __future__ import print_function, division import weakref + import spead2 as _spead2 -from spead2._spead2.send import (StreamConfig, Heap, PacketGenerator, - BytesStream, UdpStream, TcpStream, InprocStream) +from spead2._spead2.send import ( # noqa: F401 + StreamConfig, Heap, PacketGenerator, + BytesStream, UdpStream, TcpStream, InprocStream) try: - from spead2._spead2.send import UdpIbvStream + from spead2._spead2.send import UdpIbvStream # noqa: F401 except ImportError: pass -class _ItemInfo(object): +class _ItemInfo: def __init__(self, item): self.version = None self.descriptor_cnt = None self.item = weakref.ref(item) -class HeapGenerator(object): +class HeapGenerator: """Tracks which items and item values have previously been sent and generates delta heaps. @@ -49,7 +51,9 @@ :py:meth:`get_end`. """ def __init__(self, item_group, descriptor_frequency=None, flavour=_spead2.Flavour()): - self._item_group = item_group + # Workaround to avoid creating a self-reference when it's bundled into + # the same class. + self._item_group = item_group if item_group is not self else None self._info = {} # Maps ID to _ItemInfo self._descriptor_frequency = descriptor_frequency # Counter for calls to add_to_heap. This is independent of the @@ -108,9 +112,11 @@ raise ValueError("descriptors must be one of 'stale', 'all', 'none'") if data not in ['stale', 'all', 'none']: raise ValueError("data must be one of 'stale', 'all', 'none'") - for item in self._item_group.values(): + item_group = self._item_group if self._item_group is not None else self + for item in item_group.values(): info = self._get_info(item) - if (descriptors == 'all') or (descriptors == 'stale' and self._descriptor_stale(item, info)): + if (descriptors == 'all') or (descriptors == 'stale' + and self._descriptor_stale(item, info)): heap.add_descriptor(item) info.descriptor_cnt = self._descriptor_cnt if item.value is not None: diff -Nru spead2-1.10.0/spead2/send/__init__.pyi spead2-2.1.0/spead2/send/__init__.pyi --- spead2-1.10.0/spead2/send/__init__.pyi 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/send/__init__.pyi 2019-11-13 08:46:10.000000000 +0000 @@ -0,0 +1,167 @@ +# Copyright 2019 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from typing import Text, Union, Iterator, Optional, overload +import socket + +import spead2 +from spead2 import _PybindStr + +class Heap(object): + def __init__(self, flavour: spead2.Flavour) -> None: ... + @property + def flavour(self) -> spead2.Flavour: ... + @property + def repeat_pointers(self) -> bool: ... + @repeat_pointers.setter + def repeat_pointers(self, value: bool) -> None: ... + def add_item(self, item: spead2.Item) -> None: ... + def add_descriptor(self, descriptor: spead2.Descriptor) -> None: ... + def add_start(self) -> None: ... + def add_end(self) -> None: ... + +class PacketGenerator(object): + def __init__(self, heap: Heap, cnt: int, max_packet_size: int) -> None: ... + def __iter__(self) -> Iterator[bytes]: ... + +class StreamConfig(object): + DEFAULT_MAX_PACKET_SIZE: int = ... + DEFAULT_MAX_HEAPS: int = ... + DEFAULT_BURST_SIZE: int = ... + DEFAULT_BURST_RATE_RATIO: float = ... + + def __init__(self, max_packet_size: int = ..., rate: float = ..., + burst_size: int = ..., max_heaps: int = ..., + burst_rate_ratio: float = ...) -> None: ... + + @property + def max_packet_size(self) -> int: ... + @max_packet_size.setter + def max_packet_size(self, value: int) -> None: ... + + @property + def rate(self) -> float: ... + @rate.setter + def rate(self, value: float) -> None: ... + + @property + def burst_size(self) -> int: ... + @burst_size.setter + def burst_size(self, value: int) -> None: ... + + @property + def max_heaps(self) -> int: ... + @max_heaps.setter + def max_heaps(self, value: int) -> None: ... + + @property + def burst_rate_ratio(self) -> float: ... + @burst_rate_ratio.setter + def burst_rate_ratio(self, float) -> None: ... + + @property + def burst_rate(self) -> float: ... + +class _Stream(object): + def set_cnt_sequence(self, next: int, step: int) -> None: ... + +class _SyncStream(_Stream): + def send_heap(self, heap: Heap, cnt: int = ...) -> None: ... + +class _UdpStream(object): + DEFAULT_BUFFER_SIZE: int = ... + +class UdpStream(_UdpStream, _SyncStream): + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig, + buffer_size: int, socket: socket.socket) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig = ..., + buffer_size: int = ..., interface_address: _PybindStr = ...) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig, + ttl: int) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig, + ttl: int, interface_address: _PybindStr) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig, + ttl: int, interface_index: int) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + socket: socket.socket, hostname: _PybindStr, port: int, + config: StreamConfig = ...) -> None: ... + +class _UdpIbvStream(object): + DEFAULT_BUFFER_SIZE: int = ... + DEFAULT_MAX_POLL: int = ... + +class UdpIbvStream(_UdpIbvStream, _SyncStream): + def __init__(self, thread_pool: spead2.ThreadPool, + multicast_group: _PybindStr, port: int, + config: StreamConfig, + interface_address: _PybindStr, + buffer_size: int = ..., ttl: int = ..., + comp_vector: int = ..., max_pool: int = ...) -> None: ... + +class _TcpStream(object): + DEFAULT_BUFFER_SIZE: int = ... + +class TcpStream(_TcpStream, _SyncStream): + @overload + def __init__(self, thread_pool: spead2.ThreadPool, socket: socket.socket, + config: StreamConfig = ...) -> None: ... + @overload + def __init__(self, thread_pool: spead2.ThreadPool, + hostname: _PybindStr, port: int, + config: StreamConfig = ..., + buffer_size: int = ..., interface_address: _PybindStr = ...) -> None: ... + +class _BytesStream(object): + def getvalue(self) -> bytes: ... + +class BytesStream(_BytesStream, _SyncStream): + def __init__(self, thread_pool: spead2.ThreadPool, + config: StreamConfig = ...) -> None: ... + +class _InprocStream(object): + @property + def queue(self) -> spead2.InprocQueue: ... + +class InprocStream(_InprocStream, _SyncStream): + def __init__(self, thread_pool: spead2.ThreadPool, queue: spead2.InprocQueue, + config: StreamConfig = ...) -> None: ... + +class HeapGenerator(object): + def __init__(self, item_group: spead2.ItemGroup, descriptor_frequency: Optional[int] = None, + flavour: spead2.Flavour = spead2.Flavour()) -> None: ... + def add_to_heap(self, heap: Heap, descriptors: str = ..., data: str = ...) -> Heap: ... + def get_heap(self, descriptors: str = ..., data: str = ...) -> Heap: ... + def get_start(self) -> Heap: ... + def get_end(self) -> Heap: ... + +class ItemGroup(spead2.ItemGroup, HeapGenerator): + def __init__(self, descriptor_frequency: Optional[int] = None, + flavour: spead2.Flavour = ...) -> None: ... diff -Nru spead2-1.10.0/spead2/send/trollius.py spead2-2.1.0/spead2/send/trollius.py --- spead2-1.10.0/spead2/send/trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/send/trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,216 +0,0 @@ -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -""" -Integration between spead2.send and trollius -""" -from __future__ import absolute_import -import trollius -from trollius import From, Return -import spead2.send -from spead2._spead2.send import UdpStreamAsyncio as _UdpStreamAsyncio -from spead2._spead2.send import TcpStreamAsyncio as _TcpStreamAsyncio -from spead2._spead2.send import InprocStreamAsyncio as _InprocStreamAsyncio - - -def _wrap_class(name, base_class): - class Wrapped(base_class): - def __init__(self, *args, **kwargs): - self._loop = kwargs.pop('loop', None) - super(Wrapped, self).__init__(*args, **kwargs) - if self._loop is None: - self._loop = trollius.get_event_loop() - self._active = 0 - self._last_queued_future = None - - def async_send_heap(self, heap, cnt=-1, loop=None): - """Send a heap asynchronously. Note that this is *not* a coroutine: - it returns a future. Adding the heap to the queue is done - synchronously, to ensure proper ordering. - - Parameters - ---------- - heap : :py:class:`spead2.send.Heap` - Heap to send - cnt : int, optional - Heap cnt to send (defaults to auto-incrementing) - loop : :py:class:`trollius.BaseEventLoop`, optional - Event loop to use, overriding the constructor. - """ - - if loop is None: - loop = self._loop - future = trollius.Future(loop=self._loop) - - def callback(exc, bytes_transferred): - if exc is not None: - future.set_exception(exc) - else: - future.set_result(bytes_transferred) - self._active -= 1 - if self._active == 0: - self._loop.remove_reader(self.fd) - self._last_queued_future = None # Purely to free the memory - queued = super(Wrapped, self).async_send_heap(heap, callback, cnt) - if self._active == 0: - self._loop.add_reader(self.fd, self.process_callbacks) - self._active += 1 - if queued: - self._last_queued_future = future - return future - - @trollius.coroutine - def async_flush(self): - """Asynchronously wait for all enqueued heaps to be sent. Note that - this only waits for heaps passed to :meth:`async_send_heap` prior to - this call, not ones added while waiting.""" - future = self._last_queued_future - if future is not None: - yield From(trollius.wait([future])) - - Wrapped.__name__ = name - return Wrapped - - -UdpStream = _wrap_class('UdpStream', _UdpStreamAsyncio) -UdpStream.__doc__ = \ - """SPEAD over UDP with asynchronous sends. The other constructors - defined for :py:class:`spead2.send.UdpStream` are also applicable here. - - Parameters - ---------- - thread_pool : :py:class:`spead2.ThreadPool` - Thread pool handling the I/O - hostname : str - Peer hostname - port : int - Peer port - config : :py:class:`spead2.send.StreamConfig` - Stream configuration - buffer_size : int - Socket buffer size. A warning is logged if this size cannot be set due - to OS limits. - loop : :py:class:`trollius.BaseEventLoop`, optional - Event loop to use (defaults to ``trollius.get_event_loop()``) - """ - -_TcpStreamBase = _wrap_class('TcpStream', _TcpStreamAsyncio) -class TcpStream(_TcpStreamBase): - """SPEAD over TCP with asynchronous connect and sends. - - Most users will use :py:meth:`connect` to asynchronously create a stream. - The constructor should only be used if you wish to provide your own socket - and take care of connecting yourself. - - Parameters - ---------- - thread_pool : :py:class:`spead2.ThreadPool` - Thread pool handling the I/O - socket : :py:class:`socket.socket` - TCP/IP Socket that is already connected to the remote end - config : :py:class:`spead2.send.StreamConfig` - Stream configuration - """ - - @classmethod - @trollius.coroutine - def connect(cls, *args, **kwargs): - """Open a connection. - - The arguments are the same as for the constructor of - :py:class:`spead2.send.TcpStream`. - """ - loop = kwargs.get('loop') - if loop is None: - loop = trollius.get_event_loop() - future = trollius.Future(loop=loop) - def callback(arg): - if not future.done(): - if isinstance(arg, Exception): - loop.call_soon_threadsafe(future.set_exception, arg) - else: - loop.call_soon_threadsafe(future.set_result, arg) - - stream = cls(callback, *args, **kwargs) - yield From(future) - raise Return(stream) - - -InprocStream = _wrap_class('InprocStream', _InprocStreamAsyncio) -InprocStream.__doc__ = \ - """SPEAD over reliable in-process transport. - - .. note:: - - Data may still be lost if the maximum number of in-flight heaps (set - in the stream config) is exceeded. Either set this value to more - heaps than will ever be sent (which will use unbounded memory) or be - sure to block on the futures returned before exceeding the capacity. - - Parameters - ---------- - thread_pool : :py:class:`spead2.ThreadPool` - Thread pool handling the I/O - queue : :py:class:`spead2.InprocQueue` - Queue holding the data in flight - config : :py:class:`spead2.send.StreamConfig` - Stream configuration - """ - -try: - from spead2._spead2.send import UdpIbvStreamAsyncio as _UdpIbvStreamAsyncio - - UdpIbvStream = _wrap_class('UdpIbvStream', _UdpIbvStreamAsyncio) - UdpIbvStream.__doc__ = \ - """Like :class:`UdpStream`, but using the Infiniband Verbs API. - - Parameters - ---------- - thread_pool : :py:class:`spead2.ThreadPool` - Thread pool handling the I/O - multicast_group : str - IP address (or DNS name) of multicast group to join - port : int - Peer port - config : :py:class:`spead2.send.StreamConfig` - Stream configuration - interface_address : str - IP address of network interface from which to send - buffer_size : int, optional - Buffer size - ttl : int, optional - Time-To-Live of packets - comp_vector : int, optional - Completion channel vector (interrupt) - for asynchronous operation, or - a negative value to poll continuously. Polling - should not be used if there are other users of the - thread pool. If a non-negative value is provided, it - is taken modulo the number of available completion - vectors. This allows a number of readers to be - assigned sequential completion vectors and have them - load-balanced, without concern for the number - available. - max_poll : int - Maximum number of times to poll in a row, without - waiting for an interrupt (if `comp_vector` is - non-negative) or letting other code run on the - thread (if `comp_vector` is negative). - loop : :py:class:`trollius.BaseEventLoop`, optional - Event loop to use (defaults to ``trollius.get_event_loop()``) - """ - -except ImportError: - pass diff -Nru spead2-1.10.0/spead2/test/shutdown.py spead2-2.1.0/spead2/test/shutdown.py --- spead2-1.10.0/spead2/test/shutdown.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/shutdown.py 2018-12-13 11:31:34.000000000 +0000 @@ -4,7 +4,6 @@ shutdown. """ -import sys import logging import spead2 @@ -15,7 +14,7 @@ def test_logging_shutdown(): """Spam the log with lots of messages, then terminate. - + The logging thread needs to be gracefully cleaned up. """ # Set a log level that won't actually display the messages. @@ -36,7 +35,8 @@ stream.add_udp_reader(7148) sender = spead2.send.UdpStream(spead2.ThreadPool(), 'localhost', 7148) ig = spead2.send.ItemGroup() - ig.add_item(id=None, name='test', description='test', shape=(), format=[('u', 32)], value=0xdeadbeef) + ig.add_item(id=None, name='test', description='test', + shape=(), format=[('u', 32)], value=0xdeadbeef) heap = ig.get_heap() for i in range(5): sender.send_heap(heap) diff -Nru spead2-1.10.0/spead2/test/test_common.py spead2-2.1.0/spead2/test/test_common.py --- spead2-1.10.0/spead2/test/test_common.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_common.py 2019-10-07 14:23:49.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -16,23 +16,16 @@ """Tests for parts of spead2 that are shared between send and receive""" from __future__ import division, print_function -import spead2 -import numpy as np -import six -from nose.tools import * +import numpy as np +from nose.tools import ( + assert_equal, assert_greater, assert_raises, assert_true, assert_false, + assert_is, assert_is_not, assert_is_none) -def assert_equal_typed(expected, actual, msg=None): - """Check that expected and actual compare equal *and* have the same type. - - This is used for checking that strings have the correct type (str vs - unicode in Python 2, str vs bytes in Python 3). - """ - assert_equal(expected, actual, msg) - assert_equal(type(expected), type(actual), msg) +import spead2 -class TestParseRangeList(object): +class TestParseRangeList: def test_empty(self): assert_equal([], spead2.parse_range_list('')) @@ -40,10 +33,11 @@ assert_equal([1, 2, 5], spead2.parse_range_list('1,2,5')) def test_ranges(self): - assert_equal([100, 4, 5, 6, 8, 10, 12, 13], spead2.parse_range_list('100,4-6,8,10-10,12-13')) + assert_equal([100, 4, 5, 6, 8, 10, 12, 13], + spead2.parse_range_list('100,4-6,8,10-10,12-13')) -class TestThreadPool(object): +class TestThreadPool: """Smoke tests for :py:class:`spead2.ThreadPool`. These are very simple tests, because it is not actually possible to check things like the thread affinity.""" @@ -52,11 +46,18 @@ spead2.ThreadPool(4) def test_affinity(self): + spead2.ThreadPool(3, []) spead2.ThreadPool(3, [0, 1]) spead2.ThreadPool(1, [1, 0, 2]) + def test_zero_threads(self): + with assert_raises(ValueError): + spead2.ThreadPool(0) + with assert_raises(ValueError): + spead2.ThreadPool(0, [0, 1]) + -class TestFlavour(object): +class TestFlavour: def test_bad_version(self): with assert_raises(ValueError): spead2.Flavour(3, 64, 40, 0) @@ -92,7 +93,7 @@ assert_false(flavour1 == flavour3) -class TestItem(object): +class TestItem: """Tests for :py:class:`spead2.Item`. Many of these actually test :py:class:`spead2.Descriptor`, but since the @@ -103,36 +104,36 @@ """Using a non-ASCII unicode character raises a :py:exc:`UnicodeEncodeError`.""" item1 = spead2.Item(0x1000, 'name1', 'description', - (None,), format=[('c', 8)], value=six.u('\u0200')) + (None,), format=[('c', 8)], value='\u0200') item2 = spead2.Item(0x1001, 'name2', 'description2', (), - dtype='S5', value=six.u('\u0201')) + dtype='S5', value='\u0201') assert_raises(UnicodeEncodeError, item1.to_buffer) assert_raises(UnicodeEncodeError, item2.to_buffer) def test_format_and_dtype(self): """Specifying both a format and dtype raises :py:exc:`ValueError`.""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (1, 2), format=[('c', 8)], dtype='S1') + (1, 2), format=[('c', 8)], dtype='S1') def test_no_format_or_dtype(self): """At least one of format and dtype must be specified.""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (1, 2), format=None) + (1, 2), format=None) def test_invalid_order(self): """The `order` parameter must be either 'C' or 'F'.""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (1, 2), np.int32, order='K') + (1, 2), np.int32, order='K') def test_fortran_fallback(self): """The `order` parameter must be either 'C' for legacy formats.""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (1, 2), format=[('u', 32)], order='F') + (1, 2), format=[('u', 32)], order='F') def test_empty_format(self): """Format must not be empty""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (1, 2), format=[]) + (1, 2), format=[]) def test_assign_none(self): """Changing a value back to `None` raises :py:exc:`ValueError`.""" @@ -143,27 +144,27 @@ def test_multiple_unknown(self): """Multiple unknown dimensions are not allowed.""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (5, None, 3, None), format=[('u', 32)]) + (5, None, 3, None), format=[('u', 32)]) def test_numpy_unknown(self): """Unknown dimensions are not permitted when using a numpy descriptor""" assert_raises(ValueError, spead2.Item, 0x1000, 'name', 'description', - (5, None), np.int32) + (5, None), np.int32) def test_nonascii_name(self): """Name with non-ASCII characters must fail""" with assert_raises(UnicodeEncodeError): - item = spead2.Item(0x1000, six.u('\u0200'), 'description', (), np.int32) + item = spead2.Item(0x1000, '\u0200', 'description', (), np.int32) item.to_raw(spead2.Flavour()) def test_nonascii_description(self): """Description with non-ASCII characters must fail""" with assert_raises(UnicodeEncodeError): - item = spead2.Item(0x1000, 'name', six.u('\u0200'), (), np.int32) + item = spead2.Item(0x1000, 'name', '\u0200', (), np.int32) item.to_raw(spead2.Flavour()) -class TestItemGroup(object): +class TestItemGroup: """Tests for :py:class:`spead2.ItemGroup`""" def test_allocate_id(self): diff -Nru spead2-1.10.0/spead2/test/test_passthrough_asyncio.py spead2-2.1.0/spead2/test/test_passthrough_asyncio.py --- spead2-1.10.0/spead2/test/test_passthrough_asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_passthrough_asyncio.py 2019-11-13 08:46:10.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/test/test_passthrough_asyncio.py. Do not edit. - -# Copyright 2018 SKA South Africa +# Copyright 2018-2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -16,12 +14,9 @@ # along with this program. If not, see . """Tests that data can be passed through the various async transports""" -from __future__ import division, print_function, absolute_import import socket - import asyncio - import spead2 import spead2.send import spead2.recv.asyncio @@ -31,43 +26,41 @@ class BaseTestPassthroughAsync(test_passthrough.BaseTestPassthrough): - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): self.loop = asyncio.new_event_loop() - ret = self.loop.run_until_complete(self.transmit_item_group_async(item_group, memcpy, allocator)) + ret = self.loop.run_until_complete( + self.transmit_item_group_async(item_group, memcpy, allocator, new_order)) self.loop.close() return ret - @asyncio.coroutine - def transmit_item_group_async(self, item_group, memcpy, allocator): + async def transmit_item_group_async(self, item_group, memcpy, allocator, new_order='='): thread_pool = spead2.ThreadPool(2) receiver = spead2.recv.asyncio.Stream(thread_pool, loop=self.loop) receiver.set_memcpy(memcpy) if allocator is not None: receiver.set_memory_allocator(allocator) - yield from(self.prepare_receiver(receiver)) - sender = yield from(self.prepare_sender(thread_pool)) + await self.prepare_receiver(receiver) + sender = await self.prepare_sender(thread_pool) gen = spead2.send.HeapGenerator(item_group) - yield from(sender.async_send_heap(gen.get_heap())) - yield from(sender.async_send_heap(gen.get_end())) - yield from(sender.async_flush()) + await sender.async_send_heap(gen.get_heap()) + await sender.async_send_heap(gen.get_end()) + await sender.async_flush() received_item_group = spead2.ItemGroup() while True: try: - heap = yield from(receiver.get()) + heap = await receiver.get() except spead2.Stopped: break else: - received_item_group.update(heap) - return (received_item_group) + received_item_group.update(heap, new_order) + return received_item_group class TestPassthroughUdp(BaseTestPassthroughAsync): - @asyncio.coroutine - def prepare_receiver(self, receiver): + async def prepare_receiver(self, receiver): receiver.add_udp_reader(8888, bind_hostname="localhost") - @asyncio.coroutine - def prepare_sender(self, thread_pool): + async def prepare_sender(self, thread_pool): return spead2.send.asyncio.UdpStream( thread_pool, "localhost", 8888, spead2.send.StreamConfig(rate=1e7), @@ -75,16 +68,14 @@ class TestPassthroughUdpCustomSocket(BaseTestPassthroughAsync): - @asyncio.coroutine - def prepare_receiver(self, receiver): + async def prepare_receiver(self, receiver): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.bind(('127.0.0.1', 0)) self._port = sock.getsockname()[1] receiver.add_udp_reader(sock) sock.close() - @asyncio.coroutine - def prepare_sender(self, thread_pool): + async def prepare_sender(self, thread_pool): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) stream = spead2.send.asyncio.UdpStream( thread_pool, sock, '127.0.0.1', self._port, @@ -94,20 +85,17 @@ class TestPassthroughTcp(BaseTestPassthroughAsync): - @asyncio.coroutine - def prepare_receiver(self, receiver): + async def prepare_receiver(self, receiver): receiver.add_tcp_reader(8888, bind_hostname="127.0.0.1") - @asyncio.coroutine - def prepare_sender(self, thread_pool): - sender = yield from(spead2.send.asyncio.TcpStream.connect( - thread_pool, "127.0.0.1", 8888, loop=self.loop)) - return (sender) + async def prepare_sender(self, thread_pool): + sender = await spead2.send.asyncio.TcpStream.connect( + thread_pool, "127.0.0.1", 8888, loop=self.loop) + return sender class TestPassthroughTcpCustomSocket(BaseTestPassthroughAsync): - @asyncio.coroutine - def prepare_receiver(self, receiver): + async def prepare_receiver(self, receiver): sock = socket.socket() # Prevent second iteration of the test from failing sock.bind(('127.0.0.1', 0)) @@ -116,33 +104,26 @@ receiver.add_tcp_reader(sock) sock.close() - @asyncio.coroutine - def prepare_sender(self, thread_pool): + async def prepare_sender(self, thread_pool): sock = socket.socket() - sock.setblocking(0) - yield from(self.loop.sock_connect(sock, ('127.0.0.1', self._port))) + sock.setblocking(False) + await self.loop.sock_connect(sock, ('127.0.0.1', self._port)) sender = spead2.send.asyncio.TcpStream( thread_pool, sock, loop=self.loop) sock.close() - return (sender) + return sender class TestPassthroughInproc(BaseTestPassthroughAsync): - @asyncio.coroutine - def prepare_receiver(self, receiver): + async def prepare_receiver(self, receiver): receiver.add_inproc_reader(self._queue) - @asyncio.coroutine - def prepare_sender(self, thread_pool): + async def prepare_sender(self, thread_pool): return spead2.send.asyncio.InprocStream(thread_pool, self._queue, loop=self.loop) - @asyncio.coroutine - def transmit_item_group_async(self, item_group, memcpy, allocator): + async def transmit_item_group_async(self, item_group, memcpy, allocator, new_order='='): self._queue = spead2.InprocQueue() - ret = yield from(super(TestPassthroughInproc, self).transmit_item_group_async( - item_group, memcpy, allocator)) + ret = await super(TestPassthroughInproc, self).transmit_item_group_async( + item_group, memcpy, allocator, new_order) self._queue.stop() - return (ret) -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') + return ret diff -Nru spead2-1.10.0/spead2/test/test_passthrough.py spead2-2.1.0/spead2/test/test_passthrough.py --- spead2-1.10.0/spead2/test/test_passthrough.py 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_passthrough.py 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -13,44 +13,21 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -"""Test that data can be passed over the SPEAD protocol, using the various -transports, and mixing with the old PySPEAD implementation. -""" +"""Test that data can be passed over the SPEAD protocol using the various transports.""" from __future__ import division, print_function +import os +import socket +import sys + import numpy as np -import io +import netifaces +from nose.tools import assert_equal, timed +from nose.plugins.skip import SkipTest + import spead2 import spead2.send import spead2.recv -import socket -import struct -import netifaces -from decorator import decorator -from nose.tools import * -from nose.plugins.skip import SkipTest -try: - import spead64_40 -except ImportError: - spead64_40 = None -try: - import spead64_48 -except ImportError: - spead64_48 = None -try: - from socket import if_nametoindex -except ImportError: - import ctypes - import ctypes.util - _libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) - def if_nametoindex(name): - if not isinstance(name, bytes): - name = name.encode('utf-8') - ret = _libc.if_nametoindex(name) - if ret == 0: - raise OSError(ctypes.get_errno(), 'if_nametoindex failed') - else: - return ret def _assert_items_equal(item1, item2): @@ -81,24 +58,6 @@ _assert_items_equal(item_group1[key], item_group2[key]) -@decorator -def no_legacy_send(test, *args, **kwargs): - if not args[0].is_legacy_send: - test(*args, **kwargs) - - -@decorator -def no_legacy_receive(test, *args, **kwargs): - if not args[0].is_legacy_receive: - test(*args, **kwargs) - - -@decorator -def no_legacy(test, *args, **kwargs): - if not (args[0].is_legacy_send or args[0].is_legacy_receive): - test(*args, **kwargs) - - def timed_class(cls): """Class decorator version of `nose.tools.timed`""" for key in cls.__dict__: @@ -108,19 +67,17 @@ @timed_class -class BaseTestPassthrough(object): +class BaseTestPassthrough: """Tests common to all transports and libraries""" - is_legacy_send = False - is_legacy_receive = False + is_lossy = False - def _test_item_group(self, item_group, memcpy=spead2.MEMCPY_STD, allocator=None): - received_item_group = self.transmit_item_group(item_group, memcpy, allocator) + def _test_item_group(self, item_group, memcpy=spead2.MEMCPY_STD, allocator=None, new_order='='): + received_item_group = self.transmit_item_group(item_group, memcpy, allocator, new_order) assert_item_groups_equal(item_group, received_item_group) - if not self.is_legacy_receive: - for item in received_item_group.values(): - if item.dtype is not None: - assert_equal(item.value.dtype, item.value.dtype.newbyteorder('=')) + for item in received_item_group.values(): + if item.dtype is not None: + assert_equal(item.value.dtype, item.value.dtype.newbyteorder(new_order)) def test_numpy_simple(self): """A basic array with numpy encoding""" @@ -130,10 +87,23 @@ shape=data.shape, dtype=data.dtype, value=data) self._test_item_group(ig) + def test_numpy_byteorder(self): + """A numpy array in non-native byte order""" + ig = spead2.send.ItemGroup() + data = np.array([[6, 7, 8], [10, 11, 12000]], dtype=np.dtype(np.uint16).newbyteorder()) + ig.add_item(id=0x2345, name='name', description='description', + shape=data.shape, dtype=data.dtype, value=data) + self._test_item_group(ig) + self._test_item_group(ig, new_order='|') + def test_numpy_large(self): """A numpy style array split across several packets. It also uses non-temporal copies, a custom allocator, and a memory pool, to test that those all work.""" + # macOS doesn't have a big enough socket buffer to reliably transmit + # the whole thing over UDP. + if self.is_lossy and sys.platform == 'darwin': + raise SkipTest("macOS can't reliably handle large heaps over UDP") ig = spead2.send.ItemGroup() data = np.random.randn(100, 200) ig.add_item(id=0x2345, name='name', description='description', @@ -152,13 +122,8 @@ shape=(), format=format, value=data) self._test_item_group(ig) - @no_legacy_receive def test_string(self): - """Byte string is converted to array of characters and back. - - It is disabled for PySPEAD receive because PySPEAD requires a - non-standard 's' conversion to do this correctly. - """ + """Byte string is converted to array of characters and back.""" ig = spead2.send.ItemGroup() format = [('c', 8)] data = 'Hello world' @@ -166,13 +131,9 @@ shape=(None,), format=format, value=data) self._test_item_group(ig) - @no_legacy_receive def test_fallback_array_partial_bytes_small(self): """An array which takes a fractional number of bytes per element and is small enough to encode in an immediate. - - It is disabled for PySPEAD receive because PySPEAD does not decode - such items in the same way as it encodes them. """ ig = spead2.send.ItemGroup() format = [('u', 7)] @@ -181,7 +142,6 @@ shape=(len(data),), format=format, value=data) self._test_item_group(ig) - @no_legacy def test_fallback_types(self): """An array structure using a mix of types.""" ig = spead2.send.ItemGroup() @@ -191,10 +151,8 @@ shape=(2,), format=format, value=data) self._test_item_group(ig) - @no_legacy def test_numpy_fallback_struct(self): - """A structure specified using a format, but which is encodable using - numpy.""" + """A structure specified using a format, but which is encodable using numpy.""" ig = spead2.send.ItemGroup() format = [('u', 8), ('f', 32)] data = (12, 1.5) @@ -202,10 +160,8 @@ shape=(), format=format, value=data) self._test_item_group(ig) - @no_legacy def test_fallback_struct_partial_bytes(self): - """A structure which takes a fractional number of bytes per element. - """ + """A structure which takes a fractional number of bytes per element.""" ig = spead2.send.ItemGroup() format = [('u', 4), ('f', 64)] data = (12, 1.5) @@ -214,11 +170,7 @@ self._test_item_group(ig) def test_fallback_scalar(self): - """Send a scalar using fallback format descriptor. - - PySPEAD has a bug that makes this fail if the format is upgraded to a - numpy descriptor. - """ + """Send a scalar using fallback format descriptor.""" ig = spead2.send.ItemGroup() format = [('f', 64)] data = 1.5 @@ -226,7 +178,7 @@ shape=(), format=format, value=data) self._test_item_group(ig) - def test_many_tems(self): + def test_many_items(self): """Sends many items. The implementation handles few-item heaps differently (for @@ -239,7 +191,7 @@ shape=(), format=[('u', 40)], value=0x12345 * i) self._test_item_group(ig) - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): """Transmit `item_group` over the chosen transport, and return the item group received at the other end. Subclasses should override this. @@ -256,7 +208,7 @@ sender.send_heap(gen.get_end()) received_item_group = spead2.ItemGroup() for heap in receiver: - received_item_group.update(heap) + received_item_group.update(heap, new_order) return received_item_group def prepare_receiver(self, receiver): @@ -273,7 +225,7 @@ def check_ipv6(cls): if not socket.has_ipv6: raise SkipTest('platform does not support IPv6') - # Travis' Trusty image fails to bind to an IPv6 address + # Travis build systems fail to bind to an IPv6 address sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) try: sock.bind(("::1", 8888)) @@ -282,12 +234,15 @@ finally: sock.close() - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): self.check_ipv6() - return super(BaseTestPassthroughIPv6, self).transmit_item_group(item_group, memcpy, allocator) + return super().transmit_item_group( + item_group, memcpy, allocator, new_order) class TestPassthroughUdp(BaseTestPassthrough): + is_lossy = True + def prepare_receiver(self, receiver): receiver.add_udp_reader(8888, bind_hostname="localhost") @@ -299,6 +254,8 @@ class TestPassthroughUdp6(BaseTestPassthroughIPv6): + is_lossy = True + def prepare_receiver(self, receiver): receiver.add_udp_reader(8888, bind_hostname="::1") @@ -310,6 +267,8 @@ class TestPassthroughUdpCustomSocket(BaseTestPassthrough): + is_lossy = True + def prepare_receiver(self, receiver): recv_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) recv_sock.bind(("127.0.0.1", 0)) @@ -327,6 +286,7 @@ class TestPassthroughUdpMulticast(BaseTestPassthrough): + is_lossy = True MCAST_GROUP = '239.255.88.88' INTERFACE_ADDRESS = '127.0.0.1' @@ -350,7 +310,7 @@ addrs = netifaces.ifaddresses(iface).get(netifaces.AF_INET6, []) for addr in addrs: if addr['addr'] != '::1': - return if_nametoindex(iface) + return socket.if_nametoindex(iface) raise SkipTest('could not find suitable interface for test') def prepare_receiver(self, receiver): @@ -360,9 +320,40 @@ def prepare_sender(self, thread_pool): interface_index = self.get_interface_index() return spead2.send.UdpStream( - thread_pool, self.MCAST_GROUP, 8887, - spead2.send.StreamConfig(rate=1e7), - buffer_size=0, ttl=0, interface_index=interface_index) + thread_pool, self.MCAST_GROUP, 8887, + spead2.send.StreamConfig(rate=1e7), + buffer_size=0, ttl=0, interface_index=interface_index) + + +class TestPassthroughUdpIbv(BaseTestPassthrough): + is_lossy = True + MCAST_GROUP = '239.255.88.88' + + def _interface_address(self): + ifaddr = os.getenv('SPEAD2_TEST_IBV_INTERFACE_ADDRESS') + if not ifaddr: + raise SkipTest('Envar SPEAD2_TEST_IBV_INTERFACE_ADDRESS not set') + return ifaddr + + def setup(self): + # mlx5 drivers only enable multicast loopback if there are multiple + # device contexts. The sender and receiver end up sharing one, so we + # need to explicitly create another. + if not hasattr(spead2, 'IbvContext'): + raise SkipTest('IBV support not compiled in') + self._extra_context = spead2.IbvContext(self._interface_address()) + + def teardown(self): + self._extra_context.reset() + + def prepare_receiver(self, receiver): + receiver.add_udp_ibv_reader([(self.MCAST_GROUP, 8886)], self._interface_address()) + + def prepare_sender(self, thread_pool): + return spead2.send.UdpIbvStream( + thread_pool, self.MCAST_GROUP, 8886, + spead2.send.StreamConfig(rate=1e7), + self._interface_address()) class TestPassthroughTcp(BaseTestPassthrough): @@ -396,7 +387,7 @@ class TestPassthroughMem(BaseTestPassthrough): - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): thread_pool = spead2.ThreadPool(2) sender = spead2.send.BytesStream(thread_pool) gen = spead2.send.HeapGenerator(item_group) @@ -409,7 +400,7 @@ receiver.add_buffer_reader(sender.getvalue()) received_item_group = spead2.ItemGroup() for heap in receiver: - received_item_group.update(heap) + received_item_group.update(heap, new_order) return received_item_group @@ -420,16 +411,17 @@ def prepare_sender(self, thread_pool): return spead2.send.InprocStream(thread_pool, self._queue) - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): self._queue = spead2.InprocQueue() - ret = super(TestPassthroughInproc, self).transmit_item_group(item_group, memcpy, allocator) + ret = super().transmit_item_group( + item_group, memcpy, allocator, new_order) self._queue.stop() return ret class TestAllocators(BaseTestPassthrough): """Like TestPassthroughMem, but uses some custom allocators""" - def transmit_item_group(self, item_group, memcpy, allocator): + def transmit_item_group(self, item_group, memcpy, allocator, new_order='='): thread_pool = spead2.ThreadPool(2) sender = spead2.send.BytesStream(thread_pool) gen = spead2.send.HeapGenerator(item_group) @@ -442,104 +434,5 @@ receiver.add_buffer_reader(sender.getvalue()) received_item_group = spead2.ItemGroup() for heap in receiver: - received_item_group.update(heap) - return received_item_group - - -class BaseTestPassthroughLegacySend(BaseTestPassthrough): - is_legacy_send = True - - def transmit_item_group(self, item_group, memcpy, allocator): - if not self.spead: - raise SkipTest('spead module not importable') - transport = io.BytesIO() - sender = self.spead.Transmitter(transport) - legacy_item_group = self.spead.ItemGroup() - for item in item_group.values(): - # PySPEAD only supports either 1D variable or fixed-size - if item.is_variable_size(): - assert len(item.shape) == 1 - shape = -1 - else: - shape = item.shape - legacy_item_group.add_item( - id=item.id, - name=item.name, - description=item.description, - shape=shape, - fmt=self.spead.mkfmt(*item.format) if item.format else self.spead.DEFAULT_FMT, - ndarray=np.array(item.value) if not item.format else None) - legacy_item_group[item.name] = item.value - sender.send_heap(legacy_item_group.get_heap()) - sender.end() - thread_pool = spead2.ThreadPool(1) - receiver = spead2.recv.Stream(thread_pool, bug_compat=spead2.BUG_COMPAT_PYSPEAD_0_5_2) - receiver.set_memcpy(memcpy) - if allocator is not None: - receiver.set_memory_allocator(allocator) - receiver.add_buffer_reader(transport.getvalue()) - received_item_group = spead2.ItemGroup() - for heap in receiver: - received_item_group.update(heap) - return received_item_group - - -class TestPassthroughLegacySend64_40(BaseTestPassthroughLegacySend): - spead = spead64_40 - - -class TestPassthroughLegacySend64_48(BaseTestPassthroughLegacySend): - spead = spead64_48 - - -class BaseTestPassthroughLegacyReceive(BaseTestPassthrough): - is_legacy_receive = True - - def transmit_item_group(self, item_group, memcpy, allocator): - if not self.spead: - raise SkipTest('spead module not importable') - thread_pool = spead2.ThreadPool(1) - sender = spead2.send.BytesStream(thread_pool) - gen = spead2.send.HeapGenerator(item_group, flavour=self.flavour) - sender.send_heap(gen.get_heap()) - sender.send_heap(gen.get_end()) - receiver = self.spead.TransportString(sender.getvalue()) - legacy_item_group = self.spead.ItemGroup() - for heap in self.spead.iterheaps(receiver): - legacy_item_group.update(heap) - received_item_group = spead2.ItemGroup() - for key in legacy_item_group.keys(): - item = legacy_item_group.get_item(key) - # PySPEAD indicates 1D variable as -1 (scalar), everything else is fixed-sized - if item.shape == -1: - shape = (None,) - else: - shape = item.shape - if item.dtype is None: - received_item_group.add_item( - id=item.id, - name=item.name, - description=item.description, - shape=shape, - format=list(self.spead.parsefmt(item.format)), - value=item.get_value()) - else: - received_item_group.add_item( - id=item.id, - name=item.name, - description=item.description, - shape=shape, - dtype=item.dtype, - order='F' if item.fortran_order else 'C', - value=item.get_value()) + received_item_group.update(heap, new_order) return received_item_group - - -class TestPassthroughLegacyReceive64_40(BaseTestPassthroughLegacyReceive): - spead = spead64_40 - flavour = spead2.Flavour(4, 64, 40, spead2.BUG_COMPAT_PYSPEAD_0_5_2) - - -class TestPassthroughLegacyReceive64_48(BaseTestPassthroughLegacyReceive): - spead = spead64_48 - flavour = spead2.Flavour(4, 64, 48, spead2.BUG_COMPAT_PYSPEAD_0_5_2) diff -Nru spead2-1.10.0/spead2/test/test_passthrough_trollius.py spead2-2.1.0/spead2/test/test_passthrough_trollius.py --- spead2-1.10.0/spead2/test/test_passthrough_trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_passthrough_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,143 +0,0 @@ -# Copyright 2018 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Tests that data can be passed through the various async transports""" -from __future__ import division, print_function, absolute_import -import socket - -import trollius -from trollius import From, Return - -import spead2 -import spead2.send -import spead2.recv.trollius -import spead2.send.trollius - -from . import test_passthrough - - -class BaseTestPassthroughAsync(test_passthrough.BaseTestPassthrough): - def transmit_item_group(self, item_group, memcpy, allocator): - self.loop = trollius.new_event_loop() - ret = self.loop.run_until_complete(self.transmit_item_group_async(item_group, memcpy, allocator)) - self.loop.close() - return ret - - @trollius.coroutine - def transmit_item_group_async(self, item_group, memcpy, allocator): - thread_pool = spead2.ThreadPool(2) - receiver = spead2.recv.trollius.Stream(thread_pool, loop=self.loop) - receiver.set_memcpy(memcpy) - if allocator is not None: - receiver.set_memory_allocator(allocator) - yield From(self.prepare_receiver(receiver)) - sender = yield From(self.prepare_sender(thread_pool)) - gen = spead2.send.HeapGenerator(item_group) - yield From(sender.async_send_heap(gen.get_heap())) - yield From(sender.async_send_heap(gen.get_end())) - yield From(sender.async_flush()) - received_item_group = spead2.ItemGroup() - while True: - try: - heap = yield From(receiver.get()) - except spead2.Stopped: - break - else: - received_item_group.update(heap) - raise Return(received_item_group) - - -class TestPassthroughUdp(BaseTestPassthroughAsync): - @trollius.coroutine - def prepare_receiver(self, receiver): - receiver.add_udp_reader(8888, bind_hostname="localhost") - - @trollius.coroutine - def prepare_sender(self, thread_pool): - return spead2.send.trollius.UdpStream( - thread_pool, "localhost", 8888, - spead2.send.StreamConfig(rate=1e7), - buffer_size=0, loop=self.loop) - - -class TestPassthroughUdpCustomSocket(BaseTestPassthroughAsync): - @trollius.coroutine - def prepare_receiver(self, receiver): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.bind(('127.0.0.1', 0)) - self._port = sock.getsockname()[1] - receiver.add_udp_reader(sock) - sock.close() - - @trollius.coroutine - def prepare_sender(self, thread_pool): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - stream = spead2.send.trollius.UdpStream( - thread_pool, sock, '127.0.0.1', self._port, - spead2.send.StreamConfig(rate=1e7), loop=self.loop) - sock.close() - return stream - - -class TestPassthroughTcp(BaseTestPassthroughAsync): - @trollius.coroutine - def prepare_receiver(self, receiver): - receiver.add_tcp_reader(8888, bind_hostname="127.0.0.1") - - @trollius.coroutine - def prepare_sender(self, thread_pool): - sender = yield From(spead2.send.trollius.TcpStream.connect( - thread_pool, "127.0.0.1", 8888, loop=self.loop)) - raise Return(sender) - - -class TestPassthroughTcpCustomSocket(BaseTestPassthroughAsync): - @trollius.coroutine - def prepare_receiver(self, receiver): - sock = socket.socket() - # Prevent second iteration of the test from failing - sock.bind(('127.0.0.1', 0)) - self._port = sock.getsockname()[1] - sock.listen(1) - receiver.add_tcp_reader(sock) - sock.close() - - @trollius.coroutine - def prepare_sender(self, thread_pool): - sock = socket.socket() - sock.setblocking(0) - yield From(self.loop.sock_connect(sock, ('127.0.0.1', self._port))) - sender = spead2.send.trollius.TcpStream( - thread_pool, sock, loop=self.loop) - sock.close() - raise Return(sender) - - -class TestPassthroughInproc(BaseTestPassthroughAsync): - @trollius.coroutine - def prepare_receiver(self, receiver): - receiver.add_inproc_reader(self._queue) - - @trollius.coroutine - def prepare_sender(self, thread_pool): - return spead2.send.trollius.InprocStream(thread_pool, self._queue, loop=self.loop) - - @trollius.coroutine - def transmit_item_group_async(self, item_group, memcpy, allocator): - self._queue = spead2.InprocQueue() - ret = yield From(super(TestPassthroughInproc, self).transmit_item_group_async( - item_group, memcpy, allocator)) - self._queue.stop() - raise Return(ret) diff -Nru spead2-1.10.0/spead2/test/test_recv_asyncio.py spead2-2.1.0/spead2/test/test_recv_asyncio.py --- spead2-1.10.0/spead2/test/test_recv_asyncio.py 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_recv_asyncio.py 2019-09-17 20:08:17.000000000 +0000 @@ -0,0 +1,43 @@ +# Copyright 2018 SKA South Africa +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from __future__ import division, print_function + +import asynctest +from nose.tools import assert_equal, assert_true + +import spead2 +import spead2.send +import spead2.recv.asyncio + + +class TestRecvAsyncio(asynctest.TestCase): + async def test_async_iter(self): + tp = spead2.ThreadPool() + queue = spead2.InprocQueue() + sender = spead2.send.InprocStream(tp, queue) + ig = spead2.send.ItemGroup() + sender.send_heap(ig.get_start()) + sender.send_heap(ig.get_end()) + queue.stop() + heaps = [] + receiver = spead2.recv.asyncio.Stream(tp) + receiver.stop_on_stop_item = False + receiver.add_inproc_reader(queue) + async for heap in receiver: + heaps.append(heap) + assert_equal(2, len(heaps)) + assert_true(heaps[0].is_start_of_stream()) + assert_true(heaps[1].is_end_of_stream()) diff -Nru spead2-1.10.0/spead2/test/test_recv_asyncio_py35.py spead2-2.1.0/spead2/test/test_recv_asyncio_py35.py --- spead2-1.10.0/spead2/test/test_recv_asyncio_py35.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_recv_asyncio_py35.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,43 +0,0 @@ -# Copyright 2018 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -from __future__ import division, print_function - -import asynctest -from nose.tools import assert_equal, assert_true - -import spead2 -import spead2.send -import spead2.recv.asyncio - - -class TestRecvAsyncio(asynctest.TestCase): - async def test_async_iter(self): - tp = spead2.ThreadPool() - queue = spead2.InprocQueue() - sender = spead2.send.InprocStream(tp, queue) - ig = spead2.send.ItemGroup() - sender.send_heap(ig.get_start()) - sender.send_heap(ig.get_end()) - queue.stop() - heaps = [] - receiver = spead2.recv.asyncio.Stream(tp) - receiver.stop_on_stop_item = False - receiver.add_inproc_reader(queue) - async for heap in receiver: - heaps.append(heap) - assert_equal(2, len(heaps)) - assert_true(heaps[0].is_start_of_stream()) - assert_true(heaps[1].is_end_of_stream()) diff -Nru spead2-1.10.0/spead2/test/test_recv.py spead2-2.1.0/spead2/test/test_recv.py --- spead2-1.10.0/spead2/test/test_recv.py 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_recv.py 2019-10-07 14:23:49.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2015, 2017 SKA South Africa +# Copyright 2015, 2017, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -19,17 +19,16 @@ import time import numpy as np -import six -from nose.tools import * +from nose.tools import ( + assert_equal, assert_in, assert_is_instance, + assert_true, assert_false, assert_raises, assert_logs) import spead2 import spead2.recv as recv import spead2.send as send -from .test_common import assert_equal_typed - -class Item(object): +class Item: def __init__(self, id, value, immediate=False, offset=0): self.id = id self.value = value @@ -43,7 +42,7 @@ return struct.pack('>Q', (self.id << heap_address_bits) | self.offset) -class Flavour(object): +class Flavour: def __init__(self, heap_address_bits, bug_compat=0): self.heap_address_bits = heap_address_bits self.bug_compat = bug_compat @@ -119,7 +118,7 @@ if not item.immediate: value = bytes(item.value) all_items.append(Item(item.id, value, offset=offset)) - payload[offset : offset+len(value)] = value + payload[offset : offset + len(value)] = value offset += len(value) else: all_items.append(item) @@ -172,13 +171,13 @@ else: raise ValueError('Array must be C or Fortran-order contiguous') return self.make_numpy_descriptor( - id, name, description, array.dtype, array.shape, fortran_order) + id, name, description, array.dtype, array.shape, fortran_order) FLAVOUR = Flavour(48) -class TestDecode(object): +class TestDecode: """Various types of descriptors must be correctly interpreted to decode data""" def __init__(self): @@ -191,9 +190,12 @@ """ thread_pool = spead2.ThreadPool() stop_on_stop_item = kwargs.pop('stop_on_stop_item', None) + allow_unsized_heaps = kwargs.pop('allow_unsized_heaps', None) stream = recv.Stream(thread_pool, self.flavour.bug_compat, **kwargs) if stop_on_stop_item is not None: stream.stop_on_stop_item = stop_on_stop_item + if allow_unsized_heaps is not None: + stream.allow_unsized_heaps = allow_unsized_heaps stream.add_buffer_reader(data) return list(stream) @@ -206,7 +208,7 @@ ig = spead2.ItemGroup() ig.update(heaps[0]) for name, item in ig.items(): - assert_equal_typed(name, item.name) + assert_equal(name, item.name) return ig def data_to_item(self, data, expected_id): @@ -258,7 +260,6 @@ ]) ig = self.data_to_ig(packet) assert_equal(3, len(ig)) - item = ig[0x1234] assert_equal(0x9234567890AB, ig[0x1234].value) assert_equal(0x1234567890AB, ig[0x1235].value) assert_equal(-0x6DCBA9876F55, ig[0x1236].value) @@ -272,7 +273,7 @@ Item(0x1234, 'Hello world'.encode('ascii')) ]) item = self.data_to_item(packet, 0x1234) - assert_equal_typed('Hello world', item.value) + assert_equal('Hello world', item.value) def test_array(self): packet = self.flavour.make_packet_heap( @@ -484,7 +485,7 @@ ig = spead2.ItemGroup() ig.update(heaps[0]) ig.update(heaps[1]) - assert_equal_typed('Hello world', ig['test_string'].value) + assert_equal('Hello world', ig['test_string'].value) def test_size_mismatch(self): packet = self.flavour.make_packet_heap( @@ -514,7 +515,22 @@ heaps = self.data_to_heaps(packet) assert_equal(1, len(heaps)) ig = spead2.ItemGroup() - assert_raises(TypeError, ig.update, heaps[0]) + assert_raises(ValueError, ig.update, heaps[0]) + + def test_numpy_zero_size(self): + """numpy dtypes can represent zero bytes.""" + dtype = np.dtype(np.str_) + packet = self.flavour.make_packet_heap( + 1, + [ + self.flavour.make_numpy_descriptor( + 0x1234, 'empty', 'an item with zero-byte dtype', dtype, (5,)), + Item(0x1234, b'') + ]) + heaps = self.data_to_heaps(packet) + assert_equal(1, len(heaps)) + ig = spead2.ItemGroup() + assert_raises(ValueError, ig.update, heaps[0]) def test_numpy_malformed(self): """Malformed numpy header must raise :py:exc:`ValueError`.""" @@ -529,16 +545,16 @@ assert_equal(1, len(heaps)) ig = spead2.ItemGroup() assert_raises(ValueError, ig.update, heaps[0]) - helper("{'descr': 'S1'") # Syntax error: no closing brace - helper("123") # Not a dictionary - helper("import os") # Security check - helper("{'descr': 'S1'}") # Missing keys + helper("{'descr': 'S1'") # Syntax error: no closing brace + helper("123") # Not a dictionary + helper("import os") # Security check + helper("{'descr': 'S1'}") # Missing keys helper("{'descr': 'S1', 'fortran_order': False, 'shape': (), 'foo': 'bar'}") # Extra keys - helper("{'descr': 'S1', 'fortran_order': False, 'shape': (-1,)}") # Bad shape - helper("{'descr': 1, 'fortran_order': False, 'shape': ()}") # Bad descriptor - helper("{'descr': '+-', 'fortran_order': False, 'shape': ()}") # Bad descriptor - helper("{'descr': 'S1', 'fortran_order': 0, 'shape': ()}") # Bad fortran_order - helper("{'descr': 'S1', 'fortran_order': False, 'shape': (None,)}") # Bad shape + helper("{'descr': 'S1', 'fortran_order': False, 'shape': (-1,)}") # Bad shape + helper("{'descr': 1, 'fortran_order': False, 'shape': ()}") # Bad descriptor + helper("{'descr': '+-', 'fortran_order': False, 'shape': ()}") # Bad descriptor + helper("{'descr': 'S1', 'fortran_order': 0, 'shape': ()}") # Bad fortran_order + helper("{'descr': 'S1', 'fortran_order': False, 'shape': (None,)}") # Bad shape def test_nonascii_value(self): """Receiving non-ASCII characters in a c8 string must raise @@ -548,7 +564,7 @@ [ self.flavour.make_plain_descriptor( 0x1234, 'test_string', 'a byte string', [('c', 8)], [None]), - Item(0x1234, six.u('\u0200').encode('utf-8')) + Item(0x1234, '\u0200'.encode()) ]) heaps = self.data_to_heaps(packet) ig = spead2.ItemGroup() @@ -605,6 +621,22 @@ assert_equal(1, len(raw_items)) assert_equal(payload1 + payload2, bytearray(raw_items[0])) + def test_disallow_unsized_heaps(self): + """Packets without heap length rejected if disallowed""" + packet = self.flavour.make_packet( + [ + Item(spead2.HEAP_CNT_ID, 1, True), + Item(0x1000, None, False, offset=0), + Item(spead2.PAYLOAD_OFFSET_ID, 0, True), + Item(spead2.PAYLOAD_LENGTH_ID, 64, True) + ], bytes(np.arange(0, 64, dtype=np.uint8).data)) + with assert_logs('spead2', 'INFO') as cm: + heaps = self.data_to_heaps(packet, allow_unsized_heaps=False) + # Logging is asynchronous, so we have to give it a bit of time + time.sleep(0.1) + assert_equal(cm.output, ['INFO:spead2:packet rejected because it has no HEAP_LEN']) + assert_equal(0, len(heaps)) + def test_bad_offset(self): """Heap with out-of-range offset should be dropped""" packet = self.flavour.make_packet( @@ -619,7 +651,7 @@ assert_equal(0, len(heaps)) -class TestStream(object): +class TestStream: """Tests for the stream API""" def __init__(self): @@ -686,7 +718,7 @@ assert_equal(0, stats.worker_blocked) -class TestUdpReader(object): +class TestUdpReader: def test_out_of_range_udp_port(self): receiver = spead2.recv.Stream(spead2.ThreadPool()) assert_raises(TypeError, receiver.add_udp_reader, 100000) @@ -696,7 +728,7 @@ assert_raises(RuntimeError, receiver.add_udp_reader, 22) -class TestTcpReader(object): +class TestTcpReader: def setup(self): self.receiver = spead2.recv.Stream(spead2.ThreadPool()) recv_sock = socket.socket() @@ -735,7 +767,7 @@ ig = spead2.ItemGroup() ig.update(heaps[0]) for name, item in ig.items(): - assert_equal_typed(name, item.name) + assert_equal(name, item.name) return ig def data_to_item(self, data, expected_id): diff -Nru spead2-1.10.0/spead2/test/test_send_asyncio.py spead2-2.1.0/spead2/test/test_send_asyncio.py --- spead2-1.10.0/spead2/test/test_send_asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_send_asyncio.py 2019-11-13 08:46:10.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/test/test_send_asyncio.py. Do not edit. - -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -15,18 +13,18 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -from __future__ import division, print_function import asyncio import numpy as np +from nose.tools import assert_greater, assert_equal, assert_raises + import spead2 import spead2.send import spead2.send.asyncio from spead2.send.asyncio import UdpStream -from nose.tools import * -class TestUdpStream(object): +class TestUdpStream: def setup(self): # Make a stream slow enough that we can test async interactions config = spead2.send.StreamConfig(rate=5e6) @@ -36,10 +34,9 @@ self.ig['test'].value = np.zeros((256 * 1024,), np.uint8) self.heap = self.ig.get_heap() - @asyncio.coroutine - def _test_async_flush(self): + async def _test_async_flush(self): assert_greater(self.stream._active, 0) - yield from(self.stream.async_flush()) + await self.stream.async_flush() assert_equal(self.stream._active, 0) def test_async_flush(self): @@ -59,9 +56,9 @@ # test needs to be run from inside the event loop asyncio.get_event_loop().run_until_complete(self._test_async_flush()) - def _test_send_error(self, future): + async def _test_send_error(self, future): with assert_raises(IOError): - yield from(future) + await future def test_send_error(self): """An error in sending must be reported through the future.""" @@ -74,14 +71,11 @@ asyncio.get_event_loop().run_until_complete(self._test_send_error(future)) -class TestTcpStream(object): - def _test_connect_failed(self): +class TestTcpStream: + async def _test_connect_failed(self): thread_pool = spead2.ThreadPool() - with assert_raises(IOError) as cm: - yield from(spead2.send.asyncio.TcpStream.connect(thread_pool, '127.0.0.1', 8887)) + with assert_raises(IOError): + await spead2.send.asyncio.TcpStream.connect(thread_pool, '127.0.0.1', 8887) def test_connect_failed(self): asyncio.get_event_loop().run_until_complete(self._test_connect_failed()) -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') diff -Nru spead2-1.10.0/spead2/test/test_send.py spead2-2.1.0/spead2/test/test_send.py --- spead2-1.10.0/spead2/test/test_send.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_send.py 2019-10-07 14:23:49.000000000 +0000 @@ -1,4 +1,4 @@ -# Copyright 2015 SKA South Africa +# Copyright 2015, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -14,16 +14,20 @@ # along with this program. If not, see . from __future__ import division, print_function -import spead2 -import spead2.send as send -import struct import binascii -import numpy as np -import weakref -import threading -import time import gc -from nose.tools import * +import struct +import time +import threading +import weakref + +import numpy as np +from nose.tools import ( + assert_equal, assert_is_none, assert_is_not_none, assert_false, + assert_raises) + +import spead2 +import spead2.send as send def hexlify(data): @@ -33,9 +37,10 @@ chunks = [] for i in range(0, len(data), 8): part = data[i : min(i + 8, len(data))] - chunks.append(b':'.join([binascii.hexlify(part[i : i+1]) for i in range(len(part))])) + chunks.append(b':'.join([binascii.hexlify(part[i : i + 1]) for i in range(len(part))])) return b' '.join(chunks) + def encode_be(size, value): """Encodes `value` as big-endian in `size` bytes""" assert size <= 8 @@ -45,7 +50,7 @@ class Flavour(spead2.Flavour): def __init__(self, version, item_pointer_bits, heap_address_bits, bug_compat=0): - super(Flavour, self).__init__(version, item_pointer_bits, heap_address_bits, bug_compat) + super().__init__(version, item_pointer_bits, heap_address_bits, bug_compat) def make_header(self, num_items): address_size = self.heap_address_bits // 8 @@ -83,7 +88,7 @@ ans.append(encode_be(8 - self.heap_address_bits // 8, length)) return b''.join(ans) - def items_to_bytes(self, items, descriptors=None, max_packet_size=1500): + def items_to_bytes(self, items, descriptors=None, max_packet_size=1500, repeat_pointers=False): if descriptors is None: descriptors = items heap = send.Heap(self) @@ -91,6 +96,7 @@ heap.add_descriptor(descriptor) for item in items: heap.add_item(item) + heap.repeat_pointers = repeat_pointers gen = send.PacketGenerator(heap, 0x123456, max_packet_size) return list(gen) @@ -103,7 +109,7 @@ yield offset -class TestEncode(object): +class TestEncode: """Test heap encoding of various data""" def __init__(self): @@ -133,7 +139,7 @@ heap = send.Heap(self.flavour) heap.add_item(item) del item - packets = list(send.PacketGenerator(heap, 0x123456, 1472)) + packets = list(send.PacketGenerator(heap, 0x123456, 1472)) # noqa: F841 assert_is_not_none(weak()) del heap # pypy needs multiple gc passes to wind it all up @@ -310,7 +316,6 @@ """Sending a small item with fixed shape must use an immediate.""" id = 0x2345 data = 0x7654 - payload = struct.pack('>I', data) expected = [ b''.join([ self.flavour.make_header(6), @@ -352,6 +357,18 @@ packet = self.flavour.items_to_bytes([item], []) assert_equal(hexlify(expected), hexlify(packet)) + def test_numpy_zero_length(self): + """A zero-length numpy type raises :exc:`ValueError`""" + with assert_raises(ValueError): + spead2.Item(id=0x2345, name='name', description='description', + shape=(), dtype=np.str_) + + def test_fallback_zero_length(self): + """A zero-length type raises :exc:`ValueError`""" + with assert_raises(ValueError): + spead2.Item(id=0x2345, name='name', description='description', + shape=(), format=[('u', 0)]) + def test_start(self): """Tests sending a start-of-stream marker.""" expected = [ @@ -390,8 +407,43 @@ packet = list(send.PacketGenerator(heap, 0x123456, 1500)) assert_equal(hexlify(expected), hexlify(packet)) + def test_replicate_pointers(self): + """Tests sending a heap with replicate_pointers set to true""" + id = 0x2345 + data = np.arange(32, dtype=np.uint8) + item1 = spead2.Item(id=id, name='item1', description='addressed item', + shape=data.shape, dtype=data.dtype, value=data) + item2 = spead2.Item(id=id + 1, name='item2', description='inline item', + shape=(), format=[('u', self.flavour.heap_address_bits)], + value=0xdeadbeef) + expected = [ + b''.join([ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), + self.flavour.make_address(id, 0), + self.flavour.make_immediate(id + 1, 0xdeadbeef), + data.tobytes()[0:16] + ]), + b''.join([ + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, 0x123456), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 32), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 16), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 16), + self.flavour.make_address(id, 0), + self.flavour.make_immediate(id + 1, 0xdeadbeef), + data.tobytes()[16:32] + ]) + ] + packets = self.flavour.items_to_bytes([item1, item2], [], max_packet_size=72, + repeat_pointers=True) + assert_equal(hexlify(expected), hexlify(packets)) + -class TestStream(object): +class TestStream: def setup(self): # A slow stream, so that we can test overflowing the queue self.flavour = Flavour(4, 64, 48, 0) @@ -434,7 +486,7 @@ """An explicit set heap ID must be respected, and not increment the implicit sequence. - The implicit sequencing is also tested. + The implicit sequencing is also tested, including wrapping """ ig = send.ItemGroup(flavour=self.flavour) self.stream.send_heap(ig.get_start()) @@ -442,30 +494,40 @@ self.stream.send_heap(ig.get_start()) self.stream.send_heap(ig.get_start(), 0x9876543210ab) self.stream.send_heap(ig.get_start()) - expected_cnts = [1, 0x1111111111, 0x9876543210ab, 0x2345623456] + self.stream.set_cnt_sequence(2**48 - 1, 1) + self.stream.send_heap(ig.get_start()) + self.stream.send_heap(ig.get_start()) + expected_cnts = [1, 0x1111111111, 0x9876543210ab, 0x2345623456, + 2**48 - 1, 0] expected = b'' for cnt in expected_cnts: expected = b''.join([ - expected, - self.flavour.make_header(6), - self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), - self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), - self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), - self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), - self.flavour.make_address(spead2.NULL_ID, 0), - struct.pack('B', 0) - ]) + expected, + self.flavour.make_header(6), + self.flavour.make_immediate(spead2.HEAP_CNT_ID, cnt), + self.flavour.make_immediate(spead2.HEAP_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.PAYLOAD_OFFSET_ID, 0), + self.flavour.make_immediate(spead2.PAYLOAD_LENGTH_ID, 1), + self.flavour.make_immediate(spead2.STREAM_CTRL_ID, spead2.CTRL_STREAM_START), + self.flavour.make_address(spead2.NULL_ID, 0), + struct.pack('B', 0) + ]) assert_equal(hexlify(expected), hexlify(self.stream.getvalue())) + def test_invalid_cnt(self): + """An explicit heap ID that overflows must raise an error.""" + ig = send.ItemGroup(flavour=self.flavour) + with assert_raises(IOError): + self.stream.send_heap(ig.get_start(), 2**48) + -class TestTcpStream(object): +class TestTcpStream: def test_failed_connect(self): with assert_raises(IOError): send.TcpStream(spead2.ThreadPool(), '127.0.0.1', 8887) -class TestInprocStream(object): +class TestInprocStream: def setup(self): self.flavour = Flavour(4, 64, 48, 0) self.queue = spead2.InprocQueue() diff -Nru spead2-1.10.0/spead2/test/test_send_trollius.py spead2-2.1.0/spead2/test/test_send_trollius.py --- spead2-1.10.0/spead2/test/test_send_trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/test/test_send_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,82 +0,0 @@ -# Copyright 2015 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -from __future__ import division, print_function -import trollius -from trollius import From, Return -import numpy as np -import spead2 -import spead2.send -import spead2.send.trollius -from spead2.send.trollius import UdpStream -from nose.tools import * - - -class TestUdpStream(object): - def setup(self): - # Make a stream slow enough that we can test async interactions - config = spead2.send.StreamConfig(rate=5e6) - self.stream = UdpStream(spead2.ThreadPool(), 'localhost', 8888, config) - self.ig = spead2.send.ItemGroup() - self.ig.add_item(0x1000, 'test', 'Test item', shape=(256 * 1024,), dtype=np.uint8) - self.ig['test'].value = np.zeros((256 * 1024,), np.uint8) - self.heap = self.ig.get_heap() - - @trollius.coroutine - def _test_async_flush(self): - assert_greater(self.stream._active, 0) - yield From(self.stream.async_flush()) - assert_equal(self.stream._active, 0) - - def test_async_flush(self): - for i in range(3): - trollius.ensure_future(self.stream.async_send_heap(self.heap)) - # The above only queues up the async sends on the event loop. The rest of the - # test needs to be run from inside the event loop - trollius.get_event_loop().run_until_complete(self._test_async_flush()) - - def test_async_flush_fail(self): - """Test async_flush in the case that the last heap sent failed. - This is arranged by filling up the queue slots first. - """ - for i in range(5): - trollius.ensure_future(self.stream.async_send_heap(self.heap)) - # The above only queues up the async sends on the event loop. The rest of the - # test needs to be run from inside the event loop - trollius.get_event_loop().run_until_complete(self._test_async_flush()) - - def _test_send_error(self, future): - with assert_raises(IOError): - yield From(future) - - def test_send_error(self): - """An error in sending must be reported through the future.""" - # Create a stream with a packet size that is bigger than the likely - # MTU. It should cause an error. - stream = UdpStream( - spead2.ThreadPool(), "localhost", 8888, - spead2.send.StreamConfig(max_packet_size=100000), buffer_size=0) - future = stream.async_send_heap(self.heap) - trollius.get_event_loop().run_until_complete(self._test_send_error(future)) - - -class TestTcpStream(object): - def _test_connect_failed(self): - thread_pool = spead2.ThreadPool() - with assert_raises(IOError) as cm: - yield From(spead2.send.trollius.TcpStream.connect(thread_pool, '127.0.0.1', 8887)) - - def test_connect_failed(self): - trollius.get_event_loop().run_until_complete(self._test_connect_failed()) diff -Nru spead2-1.10.0/spead2/tools/bench_asyncio.py spead2-2.1.0/spead2/tools/bench_asyncio.py --- spead2-1.10.0/spead2/tools/bench_asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/tools/bench_asyncio.py 2019-10-07 14:23:49.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/tools/bench_asyncio.py. Do not edit. - -# Copyright 2015, 2017 SKA South Africa +# Copyright 2015, 2017, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -26,48 +24,46 @@ the master end. """ -from __future__ import division, print_function -import numpy as np -import spead2 -import spead2.recv -import spead2.recv.asyncio -import spead2.send -import spead2.send.asyncio +import sys import argparse import json -import asyncio - import collections import logging import traceback import timeit -import six +import asyncio +import numpy as np -class SlaveConnection(object): +import spead2 +import spead2.recv +import spead2.recv.asyncio +import spead2.send +import spead2.send.asyncio + + +class SlaveConnection: def __init__(self, reader, writer): self.reader = reader self.writer = writer - @asyncio.coroutine - def run_stream(self, stream): + async def run_stream(self, stream): num_heaps = 0 while True: try: - heap = yield from(stream.get()) + await stream.get() num_heaps += 1 except spead2.Stopped: - return (num_heaps) + return num_heaps def _write(self, s): self.writer.write(s.encode('ascii')) - @asyncio.coroutine - def run_control(self): + async def run_control(self): try: stream_task = None while True: - command = yield from(self.reader.readline()) + command = await self.reader.readline() command = command.decode('ascii') logging.debug("command = %s", command) if not command: @@ -78,20 +74,17 @@ logging.warning("Start received while already running: %s", command) continue args = argparse.Namespace(**command['args']) - if six.PY2: - args_dict = vars(args) - for key in args_dict: - if isinstance(args_dict[key], unicode): - args_dict[key] = args_dict[key].encode('ascii') if args.recv_affinity is not None and len(args.recv_affinity) > 0: spead2.ThreadPool.set_affinity(args.recv_affinity[0]) - thread_pool = spead2.ThreadPool(1, args.recv_affinity[1:] + args.recv_affinity[:1]) + thread_pool = spead2.ThreadPool( + 1, args.recv_affinity[1:] + args.recv_affinity[:1]) else: thread_pool = spead2.ThreadPool() thread_pool = spead2.ThreadPool() memory_pool = spead2.MemoryPool( args.heap_size, args.heap_size + 1024, args.mem_max_free, args.mem_initial) - stream = spead2.recv.asyncio.Stream(thread_pool, 0, args.heaps, args.ring_heaps) + stream = spead2.recv.asyncio.Stream(thread_pool, 0, args.heaps, + args.ring_heaps) stream.set_memory_allocator(memory_pool) if args.memcpy_nt: stream.set_memcpy(spead2.MEMCPY_NONTEMPORAL) @@ -107,7 +100,8 @@ logging.error('--recv-ibv passed but slave does not support ibv') sys.exit(1) else: - stream.add_udp_reader(args.port, args.packet, args.recv_buffer, bind_hostname) + stream.add_udp_reader(args.port, args.packet, args.recv_buffer, + bind_hostname) thread_pool = None memory_pool = None stream_task = asyncio.ensure_future(self.run_stream(stream)) @@ -117,7 +111,7 @@ logging.warning("Stop received when already stopped") continue stream.stop() - received_heaps = yield from(stream_task) + received_heaps = await stream_task self._write(json.dumps({'received_heaps': received_heaps}) + '\n') stream_task = None stream = None @@ -129,32 +123,30 @@ logging.debug("Connection closed") if stream_task is not None: stream.stop() - yield from(stream_task) + await stream_task except Exception: traceback.print_exc() -@asyncio.coroutine -def slave_connection(reader, writer): +async def slave_connection(reader, writer): try: conn = SlaveConnection(reader, writer) - yield from(conn.run_control()) + await conn.run_control() except Exception: traceback.print_exc() -def run_slave(args): - server = yield from(asyncio.start_server(slave_connection, port=args.port)) - yield from(server.wait_closed()) +async def run_slave(args): + server = await asyncio.start_server(slave_connection, port=args.port) + await server.wait_closed() -@asyncio.coroutine -def send_stream(item_group, stream, num_heaps): +async def send_stream(item_group, stream, num_heaps, args): tasks = collections.deque() transferred = 0 for i in range(num_heaps + 1): - while len(tasks) >= 2: - transferred += yield from(tasks.popleft()) + while len(tasks) >= args.heaps: + transferred += await tasks.popleft() if i == num_heaps: heap = item_group.get_end() else: @@ -162,17 +154,18 @@ task = asyncio.ensure_future(stream.async_send_heap(heap)) tasks.append(task) for task in tasks: - transferred += yield from(task) - return (transferred) + transferred += await task + return transferred -def measure_connection_once(args, rate, num_heaps, required_heaps): - reader, writer = yield from(asyncio.open_connection(args.host, args.port)) +async def measure_connection_once(args, rate, num_heaps, required_heaps): def write(s): writer.write(s.encode('ascii')) + + reader, writer = await asyncio.open_connection(args.host, args.port) write(json.dumps({'cmd': 'start', 'args': vars(args)}) + '\n') # Wait for "ready" response - response = yield from(reader.readline()) + response = await reader.readline() assert response == b'ready\n' if args.send_affinity is not None and len(args.send_affinity) > 0: spead2.ThreadPool.set_affinity(args.send_affinity[0]) @@ -184,7 +177,7 @@ max_packet_size=args.packet, burst_size=args.burst, rate=rate, - max_heaps=num_heaps + 1, + max_heaps=args.heaps, burst_rate_ratio=args.burst_rate_ratio) host = args.host if args.multicast is not None: @@ -204,44 +197,45 @@ value=np.zeros((args.heap_size,), dtype=np.uint8)) start = timeit.default_timer() - transferred = yield from(send_stream(item_group, stream, num_heaps)) + transferred = await send_stream(item_group, stream, num_heaps, args) end = timeit.default_timer() elapsed = end - start actual_rate = transferred / elapsed # Give receiver time to catch up with any queue - yield from(asyncio.sleep(0.1)) + await asyncio.sleep(0.1) write(json.dumps({'cmd': 'stop'}) + '\n') # Read number of heaps received - response = yield from(reader.readline()) + response = await reader.readline() response = json.loads(response.decode('ascii')) received_heaps = response['received_heaps'] - yield from(asyncio.sleep(0.5)) - yield from(writer.drain()) + await asyncio.sleep(0.5) + await writer.drain() writer.close() logging.debug("Received %d/%d heaps in %f seconds, rate %.3f Gbps", - received_heaps, num_heaps, elapsed, actual_rate * 8e-9) - return (received_heaps >= required_heaps, actual_rate) + received_heaps, num_heaps, elapsed, actual_rate * 8e-9) + return received_heaps >= required_heaps, actual_rate -def measure_connection(args, rate, num_heaps, required_heaps): +async def measure_connection(args, rate, num_heaps, required_heaps): good = True rate_sum = 0.0 passes = 5 for i in range(passes): - status, actual_rate = yield from(measure_connection_once(args, rate, num_heaps, required_heaps)) + status, actual_rate = await measure_connection_once(args, rate, num_heaps, + required_heaps) good = good and status rate_sum += actual_rate - return (good, rate_sum / passes) + return good, rate_sum / passes -def run_master(args): +async def run_master(args): best_actual = 0.0 # Send 1GB as fast as possible to find an upper bound - receive rate # does not matter. Also do a warmup run first to warm up the receiver. num_heaps = int(1e9 / args.heap_size) + 2 - yield from(measure_connection_once(args, 0.0, num_heaps, 0)) # warmup - good, actual_rate = yield from(measure_connection(args, 0.0, num_heaps, num_heaps - 1)) + await measure_connection_once(args, 0.0, num_heaps, 0) # warmup + good, actual_rate = await measure_connection(args, 0.0, num_heaps, num_heaps - 1) if good: if not args.quiet: print("Limited by send spead") @@ -255,7 +249,7 @@ # 1 second for warmup effects. rate = (low + high) * 0.5 num_heaps = int(max(1e9, rate) / args.heap_size) + 2 - good, actual_rate = yield from(measure_connection(args, rate, num_heaps, num_heaps - 1)) + good, actual_rate = await measure_connection(args, rate, num_heaps, num_heaps - 1) if not args.quiet: print("Rate: {:.3f} Gbps ({:.3f} actual): {}".format( rate * 8e-9, actual_rate * 8e-9, "GOOD" if good else "BAD")) @@ -276,32 +270,60 @@ parser.add_argument('--log', metavar='LEVEL', default='INFO', help='Log level [%(default)s]') subparsers = parser.add_subparsers(title='subcommands') master = subparsers.add_parser('master') - master.add_argument('--quiet', action='store_true', default=False, help='Print only the final result') - master.add_argument('--packet', metavar='BYTES', type=int, default=9172, help='Maximum packet size to use for UDP [%(default)s]') - master.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, help='Payload size for heap [%(default)s]') - master.add_argument('--addr-bits', metavar='BITS', type=int, default=40, help='Heap address bits [%(default)s]') - master.add_argument('--multicast', metavar='ADDRESS', type=str, help='Send via multicast group [unicast]') + master.add_argument('--quiet', action='store_true', default=False, + help='Print only the final result') + master.add_argument('--packet', metavar='BYTES', type=int, default=9172, + help='Maximum packet size to use for UDP [%(default)s]') + master.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, + help='Payload size for heap [%(default)s]') + master.add_argument('--addr-bits', metavar='BITS', type=int, default=40, + help='Heap address bits [%(default)s]') + master.add_argument('--multicast', metavar='ADDRESS', type=str, + help='Send via multicast group [unicast]') group = master.add_argument_group('sender options') - group.add_argument('--send-affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') - group.add_argument('--send-buffer', metavar='BYTES', type=int, default=spead2.send.asyncio.UdpStream.DEFAULT_BUFFER_SIZE, help='Socket buffer size [%(default)s]') - group.add_argument('--burst', metavar='BYTES', type=int, default=spead2.send.StreamConfig.DEFAULT_BURST_SIZE, help='Send burst size [%(default)s]') - group.add_argument('--burst-rate-ratio', metavar='RATIO', type=float, default=spead2.send.StreamConfig.DEFAULT_BURST_RATE_RATIO, help='Hard rate limit, relative to nominal rate [%(default)s]') + group.add_argument('--send-affinity', type=spead2.parse_range_list, + help='List of CPUs to pin threads to [no affinity]') + group.add_argument('--send-buffer', metavar='BYTES', type=int, + default=spead2.send.asyncio.UdpStream.DEFAULT_BUFFER_SIZE, + help='Socket buffer size [%(default)s]') + group.add_argument('--burst', metavar='BYTES', type=int, + default=spead2.send.StreamConfig.DEFAULT_BURST_SIZE, + help='Send burst size [%(default)s]') + group.add_argument('--burst-rate-ratio', metavar='RATIO', type=float, + default=spead2.send.StreamConfig.DEFAULT_BURST_RATE_RATIO, + help='Hard rate limit, relative to nominal rate [%(default)s]') if hasattr(spead2.send, 'UdpIbvStream'): - group.add_argument('--send-ibv', type=str, metavar='ADDRESS', help='Use ibverbs with this interface address [no]') - group.add_argument('--send-ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--send-ibv-max-poll', type=int, default=spead2.send.UdpIbvStream.DEFAULT_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') + group.add_argument('--send-ibv', type=str, metavar='ADDRESS', + help='Use ibverbs with this interface address [no]') + group.add_argument('--send-ibv-vector', type=int, default=0, metavar='N', + help='Completion vector, or -1 to use polling [%(default)s]') + group.add_argument('--send-ibv-max-poll', type=int, + default=spead2.send.UdpIbvStream.DEFAULT_MAX_POLL, + help='Maximum number of times to poll in a row [%(default)s]') group = master.add_argument_group('receiver options') - group.add_argument('--recv-affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') - group.add_argument('--recv-buffer', metavar='BYTES', type=int, default=spead2.recv.Stream.DEFAULT_UDP_BUFFER_SIZE, help='Socket buffer size [%(default)s]') + group.add_argument('--recv-affinity', type=spead2.parse_range_list, + help='List of CPUs to pin threads to [no affinity]') + group.add_argument('--recv-buffer', metavar='BYTES', type=int, + default=spead2.recv.Stream.DEFAULT_UDP_BUFFER_SIZE, + help='Socket buffer size [%(default)s]') if hasattr(spead2.recv.Stream, 'add_udp_ibv_reader'): - group.add_argument('--recv-ibv', type=str, metavar='ADDRESS', help='Use ibverbs with this interface address [no]') - group.add_argument('--recv-ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--recv-ibv-max-poll', type=int, default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') - group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, help='Maximum number of in-flight heaps [%(default)s]') - group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, help='Ring buffer capacity in heaps [%(default)s]') - group.add_argument('--memcpy-nt', action='store_true', help='Use non-temporal memcpy [no]') - group.add_argument('--mem-max-free', type=int, default=12, help='Maximum free memory buffers [%(default)s]') - group.add_argument('--mem-initial', type=int, default=8, help='Initial free memory buffers [%(default)s]') + group.add_argument('--recv-ibv', type=str, metavar='ADDRESS', + help='Use ibverbs with this interface address [no]') + group.add_argument('--recv-ibv-vector', type=int, default=0, metavar='N', + help='Completion vector, or -1 to use polling [%(default)s]') + group.add_argument('--recv-ibv-max-poll', type=int, + default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, + help='Maximum number of times to poll in a row [%(default)s]') + group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, + help='Maximum number of in-flight heaps [%(default)s]') + group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, + help='Ring buffer capacity in heaps [%(default)s]') + group.add_argument('--memcpy-nt', action='store_true', + help='Use non-temporal memcpy [no]') + group.add_argument('--mem-max-free', type=int, default=12, + help='Maximum free memory buffers [%(default)s]') + group.add_argument('--mem-initial', type=int, default=8, + help='Initial free memory buffers [%(default)s]') master.add_argument('host') master.add_argument('port', type=int) slave = subparsers.add_parser('slave') @@ -316,6 +338,3 @@ task = asyncio.ensure_future(task) asyncio.get_event_loop().run_until_complete(task) task.result() -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') diff -Nru spead2-1.10.0/spead2/tools/bench_trollius.py spead2-2.1.0/spead2/tools/bench_trollius.py --- spead2-1.10.0/spead2/tools/bench_trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/tools/bench_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,316 +0,0 @@ -# Copyright 2015, 2017 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Benchmark tool to estimate the sustainable SPEAD bandwidth between two -machines, for a specific set of configurations. - -Since UDP is lossy, this is not a trivial problem. We binary search for the -speed that is just sustainable. To make the test at a specific speed more -reliable, it is repeated several times, opening a new stream each time, and -with a delay to allow processors to return to idle states. A TCP control -stream is used to synchronise the two ends. All configuration is done on -the master end. -""" - -from __future__ import division, print_function -import numpy as np -import spead2 -import spead2.recv -import spead2.recv.trollius -import spead2.send -import spead2.send.trollius -import argparse -import json -import trollius -from trollius import From, Return -import collections -import logging -import traceback -import timeit -import six - - -class SlaveConnection(object): - def __init__(self, reader, writer): - self.reader = reader - self.writer = writer - - @trollius.coroutine - def run_stream(self, stream): - num_heaps = 0 - while True: - try: - heap = yield From(stream.get()) - num_heaps += 1 - except spead2.Stopped: - raise Return(num_heaps) - - def _write(self, s): - self.writer.write(s.encode('ascii')) - - @trollius.coroutine - def run_control(self): - try: - stream_task = None - while True: - command = yield From(self.reader.readline()) - command = command.decode('ascii') - logging.debug("command = %s", command) - if not command: - break - command = json.loads(command) - if command['cmd'] == 'start': - if stream_task is not None: - logging.warning("Start received while already running: %s", command) - continue - args = argparse.Namespace(**command['args']) - if six.PY2: - args_dict = vars(args) - for key in args_dict: - if isinstance(args_dict[key], unicode): - args_dict[key] = args_dict[key].encode('ascii') - if args.recv_affinity is not None and len(args.recv_affinity) > 0: - spead2.ThreadPool.set_affinity(args.recv_affinity[0]) - thread_pool = spead2.ThreadPool(1, args.recv_affinity[1:] + args.recv_affinity[:1]) - else: - thread_pool = spead2.ThreadPool() - thread_pool = spead2.ThreadPool() - memory_pool = spead2.MemoryPool( - args.heap_size, args.heap_size + 1024, args.mem_max_free, args.mem_initial) - stream = spead2.recv.trollius.Stream(thread_pool, 0, args.heaps, args.ring_heaps) - stream.set_memory_allocator(memory_pool) - if args.memcpy_nt: - stream.set_memcpy(spead2.MEMCPY_NONTEMPORAL) - bind_hostname = '' if args.multicast is None else args.multicast - if 'recv_ibv' in args and args.recv_ibv is not None: - try: - stream.add_udp_ibv_reader( - bind_hostname, args.port, - args.recv_ibv, - args.packet, args.recv_buffer, - args.recv_ibv_vector, args.recv_ibv_max_poll) - except AttributeError: - logging.error('--recv-ibv passed but slave does not support ibv') - sys.exit(1) - else: - stream.add_udp_reader(args.port, args.packet, args.recv_buffer, bind_hostname) - thread_pool = None - memory_pool = None - stream_task = trollius.ensure_future(self.run_stream(stream)) - self._write('ready\n') - elif command['cmd'] == 'stop': - if stream_task is None: - logging.warning("Stop received when already stopped") - continue - stream.stop() - received_heaps = yield From(stream_task) - self._write(json.dumps({'received_heaps': received_heaps}) + '\n') - stream_task = None - stream = None - elif command['cmd'] == 'exit': - break - else: - logging.warning("Bad command: %s", command) - continue - logging.debug("Connection closed") - if stream_task is not None: - stream.stop() - yield From(stream_task) - except Exception: - traceback.print_exc() - - -@trollius.coroutine -def slave_connection(reader, writer): - try: - conn = SlaveConnection(reader, writer) - yield From(conn.run_control()) - except Exception: - traceback.print_exc() - - -def run_slave(args): - server = yield From(trollius.start_server(slave_connection, port=args.port)) - yield From(server.wait_closed()) - - -@trollius.coroutine -def send_stream(item_group, stream, num_heaps): - tasks = collections.deque() - transferred = 0 - for i in range(num_heaps + 1): - while len(tasks) >= 2: - transferred += yield From(tasks.popleft()) - if i == num_heaps: - heap = item_group.get_end() - else: - heap = item_group.get_heap(data='all') - task = trollius.ensure_future(stream.async_send_heap(heap)) - tasks.append(task) - for task in tasks: - transferred += yield From(task) - raise Return(transferred) - - -def measure_connection_once(args, rate, num_heaps, required_heaps): - reader, writer = yield From(trollius.open_connection(args.host, args.port)) - def write(s): - writer.write(s.encode('ascii')) - write(json.dumps({'cmd': 'start', 'args': vars(args)}) + '\n') - # Wait for "ready" response - response = yield From(reader.readline()) - assert response == b'ready\n' - if args.send_affinity is not None and len(args.send_affinity) > 0: - spead2.ThreadPool.set_affinity(args.send_affinity[0]) - thread_pool = spead2.ThreadPool(1, args.send_affinity[1:] + args.send_affinity[:1]) - else: - thread_pool = spead2.ThreadPool() - thread_pool = spead2.ThreadPool() - config = spead2.send.StreamConfig( - max_packet_size=args.packet, - burst_size=args.burst, - rate=rate, - max_heaps=num_heaps + 1, - burst_rate_ratio=args.burst_rate_ratio) - host = args.host - if args.multicast is not None: - host = args.multicast - if 'send_ibv' in args and args.send_ibv is not None: - stream = spead2.send.trollius.UdpIbvStream( - thread_pool, host, args.port, config, args.send_ibv, args.send_buffer, - 1, args.send_ibv_vector, args.send_ibv_max_poll) - else: - stream = spead2.send.trollius.UdpStream( - thread_pool, host, args.port, config, args.send_buffer) - item_group = spead2.send.ItemGroup( - flavour=spead2.Flavour(4, 64, args.addr_bits, 0)) - item_group.add_item(id=None, name='Test item', - description='A test item with arbitrary value', - shape=(args.heap_size,), dtype=np.uint8, - value=np.zeros((args.heap_size,), dtype=np.uint8)) - - start = timeit.default_timer() - transferred = yield From(send_stream(item_group, stream, num_heaps)) - end = timeit.default_timer() - elapsed = end - start - actual_rate = transferred / elapsed - # Give receiver time to catch up with any queue - yield From(trollius.sleep(0.1)) - write(json.dumps({'cmd': 'stop'}) + '\n') - # Read number of heaps received - response = yield From(reader.readline()) - response = json.loads(response.decode('ascii')) - received_heaps = response['received_heaps'] - yield From(trollius.sleep(0.5)) - yield From(writer.drain()) - writer.close() - logging.debug("Received %d/%d heaps in %f seconds, rate %.3f Gbps", - received_heaps, num_heaps, elapsed, actual_rate * 8e-9) - raise Return(received_heaps >= required_heaps, actual_rate) - - -def measure_connection(args, rate, num_heaps, required_heaps): - good = True - rate_sum = 0.0 - passes = 5 - for i in range(passes): - status, actual_rate = yield From(measure_connection_once(args, rate, num_heaps, required_heaps)) - good = good and status - rate_sum += actual_rate - raise Return(good, rate_sum / passes) - - -def run_master(args): - best_actual = 0.0 - - # Send 1GB as fast as possible to find an upper bound - receive rate - # does not matter. Also do a warmup run first to warm up the receiver. - num_heaps = int(1e9 / args.heap_size) + 2 - yield From(measure_connection_once(args, 0.0, num_heaps, 0)) # warmup - good, actual_rate = yield From(measure_connection(args, 0.0, num_heaps, num_heaps - 1)) - if good: - if not args.quiet: - print("Limited by send spead") - best_actual = actual_rate - else: - print("Send rate: {:.3f} Gbps".format(actual_rate * 8e-9)) - low = 0.0 - high = actual_rate - while high - low > high * 0.02: - # Need at least 1GB of data to overwhelm cache effects, and want at least - # 1 second for warmup effects. - rate = (low + high) * 0.5 - num_heaps = int(max(1e9, rate) / args.heap_size) + 2 - good, actual_rate = yield From(measure_connection(args, rate, num_heaps, num_heaps - 1)) - if not args.quiet: - print("Rate: {:.3f} Gbps ({:.3f} actual): {}".format( - rate * 8e-9, actual_rate * 8e-9, "GOOD" if good else "BAD")) - if good: - low = rate - best_actual = actual_rate - else: - high = rate - rate_gbps = best_actual * 8e-9 - if args.quiet: - print(rate_gbps) - else: - print("Sustainable rate: {:.3f} Gbps".format(rate_gbps)) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--log', metavar='LEVEL', default='INFO', help='Log level [%(default)s]') - subparsers = parser.add_subparsers(title='subcommands') - master = subparsers.add_parser('master') - master.add_argument('--quiet', action='store_true', default=False, help='Print only the final result') - master.add_argument('--packet', metavar='BYTES', type=int, default=9172, help='Maximum packet size to use for UDP [%(default)s]') - master.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, help='Payload size for heap [%(default)s]') - master.add_argument('--addr-bits', metavar='BITS', type=int, default=40, help='Heap address bits [%(default)s]') - master.add_argument('--multicast', metavar='ADDRESS', type=str, help='Send via multicast group [unicast]') - group = master.add_argument_group('sender options') - group.add_argument('--send-affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') - group.add_argument('--send-buffer', metavar='BYTES', type=int, default=spead2.send.trollius.UdpStream.DEFAULT_BUFFER_SIZE, help='Socket buffer size [%(default)s]') - group.add_argument('--burst', metavar='BYTES', type=int, default=spead2.send.StreamConfig.DEFAULT_BURST_SIZE, help='Send burst size [%(default)s]') - group.add_argument('--burst-rate-ratio', metavar='RATIO', type=float, default=spead2.send.StreamConfig.DEFAULT_BURST_RATE_RATIO, help='Hard rate limit, relative to nominal rate [%(default)s]') - if hasattr(spead2.send, 'UdpIbvStream'): - group.add_argument('--send-ibv', type=str, metavar='ADDRESS', help='Use ibverbs with this interface address [no]') - group.add_argument('--send-ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--send-ibv-max-poll', type=int, default=spead2.send.UdpIbvStream.DEFAULT_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') - group = master.add_argument_group('receiver options') - group.add_argument('--recv-affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') - group.add_argument('--recv-buffer', metavar='BYTES', type=int, default=spead2.recv.Stream.DEFAULT_UDP_BUFFER_SIZE, help='Socket buffer size [%(default)s]') - if hasattr(spead2.recv.Stream, 'add_udp_ibv_reader'): - group.add_argument('--recv-ibv', type=str, metavar='ADDRESS', help='Use ibverbs with this interface address [no]') - group.add_argument('--recv-ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--recv-ibv-max-poll', type=int, default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') - group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, help='Maximum number of in-flight heaps [%(default)s]') - group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, help='Ring buffer capacity in heaps [%(default)s]') - group.add_argument('--memcpy-nt', action='store_true', help='Use non-temporal memcpy [no]') - group.add_argument('--mem-max-free', type=int, default=12, help='Maximum free memory buffers [%(default)s]') - group.add_argument('--mem-initial', type=int, default=8, help='Initial free memory buffers [%(default)s]') - master.add_argument('host') - master.add_argument('port', type=int) - slave = subparsers.add_parser('slave') - slave.add_argument('port', type=int) - - args = parser.parse_args() - logging.basicConfig(level=getattr(logging, args.log.upper())) - if 'host' in args: - task = run_master(args) - else: - task = run_slave(args) - task = trollius.ensure_future(task) - trollius.get_event_loop().run_until_complete(task) - task.result() diff -Nru spead2-1.10.0/spead2/tools/recv_asyncio.py spead2-2.1.0/spead2/tools/recv_asyncio.py --- spead2-1.10.0/spead2/tools/recv_asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/tools/recv_asyncio.py 2020-02-12 09:25:32.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/tools/recv_asyncio.py. Do not edit. - -# Copyright 2015, 2017-2018 SKA South Africa +# Copyright 2015, 2017-2020 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -22,15 +20,14 @@ necessary, to allow multiple code-paths to be exercised. """ -from __future__ import print_function, division -import spead2 -import spead2.recv -import spead2.recv.asyncio import logging import argparse import signal import asyncio +import spead2 +import spead2.recv +import spead2.recv.asyncio def get_args(): @@ -45,27 +42,40 @@ group = parser.add_argument_group('Protocol options') group.add_argument('--tcp', action='store_true', help='Receive data over TCP instead of UDP') - group.add_argument('--bind', type=str, default='', help='Interface address for multicast') + group.add_argument('--bind', type=str, default='', help='Interface address') group.add_argument('--pyspead', action='store_true', help='Be bug-compatible with PySPEAD') group.add_argument('--joint', action='store_true', help='Treat all sources as a single stream') - group.add_argument('--packet', type=int, default=spead2.recv.Stream.DEFAULT_UDP_MAX_SIZE, help='Maximum packet size to accept for UDP [%(default)s]') + group.add_argument('--packet', type=int, default=spead2.recv.Stream.DEFAULT_UDP_MAX_SIZE, + help='Maximum packet size to accept for UDP [%(default)s]') group = parser.add_argument_group('Performance options') group.add_argument('--buffer', type=int, help='Socket buffer size') - group.add_argument('--threads', type=int, default=1, help='Number of worker threads [%(default)s]') - group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, help='Maximum number of in-flight heaps [%(default)s]') - group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, help='Ring buffer capacity in heaps [%(default)s]') + group.add_argument('--threads', type=int, default=1, + help='Number of worker threads [%(default)s]') + group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, + help='Maximum number of in-flight heaps [%(default)s]') + group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, + help='Ring buffer capacity in heaps [%(default)s]') group.add_argument('--mem-pool', action='store_true', help='Use a memory pool') - group.add_argument('--mem-lower', type=int, default=16384, help='Minimum allocation which will use the memory pool [%(default)s]') - group.add_argument('--mem-upper', type=int, default=32 * 1024**2, help='Maximum allocation which will use the memory pool [%(default)s]') - group.add_argument('--mem-max-free', type=int, default=12, help='Maximum free memory buffers [%(default)s]') - group.add_argument('--mem-initial', type=int, default=8, help='Initial free memory buffers [%(default)s]') - group.add_argument('--memcpy-nt', action='store_true', help='Use non-temporal memcpy') - group.add_argument('--affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') + group.add_argument('--mem-lower', type=int, default=16384, + help='Minimum allocation which will use the memory pool [%(default)s]') + group.add_argument('--mem-upper', type=int, default=32 * 1024**2, + help='Maximum allocation which will use the memory pool [%(default)s]') + group.add_argument('--mem-max-free', type=int, default=12, + help='Maximum free memory buffers [%(default)s]') + group.add_argument('--mem-initial', type=int, default=8, + help='Initial free memory buffers [%(default)s]') + group.add_argument('--memcpy-nt', action='store_true', + help='Use non-temporal memcpy') + group.add_argument('--affinity', type=spead2.parse_range_list, + help='List of CPUs to pin threads to [no affinity]') if hasattr(spead2.recv.Stream, 'add_udp_ibv_reader'): group.add_argument('--ibv', action='store_true', help='Use ibverbs [no]') - group.add_argument('--ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--ibv-max-poll', type=int, default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') + group.add_argument('--ibv-vector', type=int, default=0, metavar='N', + help='Completion vector, or -1 to use polling [%(default)s]') + group.add_argument('--ibv-max-poll', type=int, + default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, + help='Maximum number of times to poll in a row [%(default)s]') args = parser.parse_args() if args.ibv and not args.bind: @@ -80,8 +90,7 @@ return args -@asyncio.coroutine -def run_stream(stream, name, args): +async def run_stream(stream, name, args): try: item_group = spead2.ItemGroup() num_heaps = 0 @@ -89,7 +98,7 @@ try: if num_heaps == args.max_heaps: break - heap = yield from(stream.get()) + heap = await stream.get() print("Received heap {} on stream {}".format(heap.cnt, name)) num_heaps += 1 try: @@ -142,15 +151,13 @@ try: stream.add_udp_pcap_file_reader(source) except AttributeError: - raise RuntimeError('spead2 was compiled without pcap support') + raise RuntimeError('spead2 was compiled without pcap support') from None else: if args.tcp: stream.add_tcp_reader(port, args.packet, args.buffer, host) elif 'ibv' in args and args.ibv: - if host is None: - raise ValueError('a multicast group is required when using --ibv') ibv_endpoints.append((host, port)) - elif args.bind and host: + elif args.bind: stream.add_udp_reader(host, port, args.packet, args.buffer, args.bind) else: stream.add_udp_reader(port, args.packet, args.buffer, host) @@ -177,7 +184,8 @@ thread_pool = spead2.ThreadPool(args.threads) memory_pool = None if args.mem_pool: - memory_pool = spead2.MemoryPool(args.mem_lower, args.mem_upper, args.mem_max_free, args.mem_initial) + memory_pool = spead2.MemoryPool(args.mem_lower, args.mem_upper, + args.mem_max_free, args.mem_initial) if args.joint: coros_and_streams = [make_coro(args.source)] else: @@ -191,6 +199,3 @@ except asyncio.CancelledError: pass loop.close() -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') diff -Nru spead2-1.10.0/spead2/tools/recv_trollius.py spead2-2.1.0/spead2/tools/recv_trollius.py --- spead2-1.10.0/spead2/tools/recv_trollius.py 2018-12-11 10:52:34.000000000 +0000 +++ spead2-2.1.0/spead2/tools/recv_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,191 +0,0 @@ -# Copyright 2015, 2017-2018 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Receive SPEAD packets and log the contents. - -This is both a tool for debugging SPEAD data flows and a demonstrator for the -spead2 package. It thus has many more command-line options than are strictly -necessary, to allow multiple code-paths to be exercised. -""" - -from __future__ import print_function, division -import spead2 -import spead2.recv -import spead2.recv.trollius -import logging -import argparse -import signal -import trollius -from trollius import From - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('source', nargs='+', help='Sources (filename, host:port or port)') - - group = parser.add_argument_group('Output options') - group.add_argument('--log', metavar='LEVEL', default='INFO', help='Log level [%(default)s]') - group.add_argument('--values', action='store_true', help='Show heap values') - group.add_argument('--descriptors', action='store_true', help='Show descriptors') - group.add_argument('--max-heaps', type=int, help='Stop receiving after this many heaps') - - group = parser.add_argument_group('Protocol options') - group.add_argument('--tcp', action='store_true', help='Receive data over TCP instead of UDP') - group.add_argument('--bind', type=str, default='', help='Interface address for multicast') - group.add_argument('--pyspead', action='store_true', help='Be bug-compatible with PySPEAD') - group.add_argument('--joint', action='store_true', help='Treat all sources as a single stream') - group.add_argument('--packet', type=int, default=spead2.recv.Stream.DEFAULT_UDP_MAX_SIZE, help='Maximum packet size to accept for UDP [%(default)s]') - - group = parser.add_argument_group('Performance options') - group.add_argument('--buffer', type=int, help='Socket buffer size') - group.add_argument('--threads', type=int, default=1, help='Number of worker threads [%(default)s]') - group.add_argument('--heaps', type=int, default=spead2.recv.Stream.DEFAULT_MAX_HEAPS, help='Maximum number of in-flight heaps [%(default)s]') - group.add_argument('--ring-heaps', type=int, default=spead2.recv.Stream.DEFAULT_RING_HEAPS, help='Ring buffer capacity in heaps [%(default)s]') - group.add_argument('--mem-pool', action='store_true', help='Use a memory pool') - group.add_argument('--mem-lower', type=int, default=16384, help='Minimum allocation which will use the memory pool [%(default)s]') - group.add_argument('--mem-upper', type=int, default=32 * 1024**2, help='Maximum allocation which will use the memory pool [%(default)s]') - group.add_argument('--mem-max-free', type=int, default=12, help='Maximum free memory buffers [%(default)s]') - group.add_argument('--mem-initial', type=int, default=8, help='Initial free memory buffers [%(default)s]') - group.add_argument('--memcpy-nt', action='store_true', help='Use non-temporal memcpy') - group.add_argument('--affinity', type=spead2.parse_range_list, help='List of CPUs to pin threads to [no affinity]') - if hasattr(spead2.recv.Stream, 'add_udp_ibv_reader'): - group.add_argument('--ibv', action='store_true', help='Use ibverbs [no]') - group.add_argument('--ibv-vector', type=int, default=0, metavar='N', help='Completion vector, or -1 to use polling [%(default)s]') - group.add_argument('--ibv-max-poll', type=int, default=spead2.recv.Stream.DEFAULT_UDP_IBV_MAX_POLL, help='Maximum number of times to poll in a row [%(default)s]') - - args = parser.parse_args() - if args.ibv and not args.bind: - parser.error('--ibv requires --bind') - if args.tcp and args.ibv: - parser.error('--ibv and --tcp are incompatible') - if args.buffer is None: - if args.tcp: - args.buffer = spead2.recv.trollius.Stream.DEFAULT_TCP_BUFFER_SIZE - else: - args.buffer = spead2.recv.trollius.Stream.DEFAULT_UDP_BUFFER_SIZE - return args - - -@trollius.coroutine -def run_stream(stream, name, args): - try: - item_group = spead2.ItemGroup() - num_heaps = 0 - while True: - try: - if num_heaps == args.max_heaps: - break - heap = yield From(stream.get()) - print("Received heap {} on stream {}".format(heap.cnt, name)) - num_heaps += 1 - try: - if args.descriptors: - for raw_descriptor in heap.get_descriptors(): - descriptor = spead2.Descriptor.from_raw(raw_descriptor, heap.flavour) - print('''\ - Descriptor for {0.name} ({0.id:#x}) - description: {0.description} - format: {0.format} - dtype: {0.dtype} - shape: {0.shape}'''.format(descriptor)) - changed = item_group.update(heap) - for (key, item) in changed.items(): - if args.values: - print(key, '=', item.value) - else: - print(key) - except ValueError as e: - print("Error raised processing heap: {}".format(e)) - except (spead2.Stopped, trollius.CancelledError): - print("Shutting down stream {} after {} heaps".format(name, num_heaps)) - stats = stream.stats - for key in dir(stats): - if not key.startswith('_'): - print("{}: {}".format(key, getattr(stats, key))) - break - finally: - stream.stop() - - -def main(): - def make_stream(sources): - bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2 if args.pyspead else 0 - stream = spead2.recv.trollius.Stream(thread_pool, bug_compat, args.heaps, args.ring_heaps) - if memory_pool is not None: - stream.set_memory_allocator(memory_pool) - if args.memcpy_nt: - stream.set_memcpy(spead2.MEMCPY_NONTEMPORAL) - ibv_endpoints = [] - for source in sources: - try: - if ':' in source: - host, port = source.rsplit(':', 1) - port = int(port) - else: - host = '' - port = int(source) - except ValueError: - try: - stream.add_udp_pcap_file_reader(source) - except AttributeError: - raise RuntimeError('spead2 was compiled without pcap support') - else: - if args.tcp: - stream.add_tcp_reader(port, args.packet, args.buffer, host) - elif 'ibv' in args and args.ibv: - if host is None: - raise ValueError('a multicast group is required when using --ibv') - ibv_endpoints.append((host, port)) - elif args.bind and host: - stream.add_udp_reader(host, port, args.packet, args.buffer, args.bind) - else: - stream.add_udp_reader(port, args.packet, args.buffer, host) - if ibv_endpoints: - stream.add_udp_ibv_reader(ibv_endpoints, args.bind, args.packet, - args.buffer, args.ibv_vector, args.ibv_max_poll) - return stream - - def make_coro(sources): - stream = make_stream(sources) - return run_stream(stream, sources[0], args), stream - - def stop_streams(): - for stream in streams: - stream.stop() - - args = get_args() - logging.basicConfig(level=getattr(logging, args.log.upper())) - - if args.affinity is not None and len(args.affinity) > 0: - spead2.ThreadPool.set_affinity(args.affinity[0]) - thread_pool = spead2.ThreadPool(args.threads, args.affinity[1:] + args.affinity[:1]) - else: - thread_pool = spead2.ThreadPool(args.threads) - memory_pool = None - if args.mem_pool: - memory_pool = spead2.MemoryPool(args.mem_lower, args.mem_upper, args.mem_max_free, args.mem_initial) - if args.joint: - coros_and_streams = [make_coro(args.source)] - else: - coros_and_streams = [make_coro([source]) for source in args.source] - coros, streams = zip(*coros_and_streams) - main_task = trollius.ensure_future(trollius.gather(*coros)) - loop = trollius.get_event_loop() - loop.add_signal_handler(signal.SIGINT, stop_streams) - try: - loop.run_until_complete(main_task) - except trollius.CancelledError: - pass - loop.close() diff -Nru spead2-1.10.0/spead2/tools/send_asyncio.py spead2-2.1.0/spead2/tools/send_asyncio.py --- spead2-1.10.0/spead2/tools/send_asyncio.py 2018-12-11 11:29:14.000000000 +0000 +++ spead2-2.1.0/spead2/tools/send_asyncio.py 2019-09-17 20:08:17.000000000 +0000 @@ -1,6 +1,4 @@ -# Generated by bootstrap.sh from spead2/tools/send_asyncio.py. Do not edit. - -# Copyright 2015, 2017 SKA South Africa +# Copyright 2015, 2017, 2019 SKA South Africa # # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU Lesser General Public License as published by the Free @@ -15,22 +13,23 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -"""Generate and send SPEAD packets. This is mainly a benchmark application, but also -demonstrates the API.""" +"""Generate and send SPEAD packets. + +This is mainly a benchmark application, but also demonstrates the API.""" -from __future__ import print_function, division -import spead2 -import spead2.send -import spead2.send.asyncio -import numpy as np import logging import sys import time import itertools import argparse -import asyncio import collections +import asyncio + +import numpy as np +import spead2 +import spead2.send +import spead2.send.asyncio def get_args(): @@ -39,34 +38,61 @@ parser.add_argument('port', type=int) group = parser.add_argument_group('Data options') - group.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, help='Payload size for heap [%(default)s]') - group.add_argument('--items', type=int, default=1, help='Number of items per heap [%(default)s]') - group.add_argument('--dtype', type=str, default='= 2: + if len(tasks) >= args.max_heaps: try: - n_bytes += yield from(tasks.popleft()) + n_bytes += await tasks.popleft() except Exception as error: n_errors += 1 last_error = error @@ -108,7 +133,7 @@ tasks.append(task) while len(tasks) > 0: try: - n_bytes += yield from(tasks.popleft()) + n_bytes += await tasks.popleft() except Exception as error: n_errors += 1 last_error = error @@ -119,8 +144,7 @@ n_bytes, elapsed, n_bytes * 8 / elapsed / 1e9)) -@asyncio.coroutine -def async_main(): +async def async_main(): args = get_args() logging.basicConfig(level=getattr(logging, args.log.upper())) @@ -148,10 +172,11 @@ max_packet_size=args.packet, burst_size=args.burst, rate=args.rate * 10**9 / 8, - burst_rate_ratio=args.burst_rate_ratio) + burst_rate_ratio=args.burst_rate_ratio, + max_heaps=args.max_heaps) if args.tcp: - stream = yield from(spead2.send.asyncio.TcpStream.connect( - thread_pool, args.host, args.port, config, args.buffer, args.bind)) + stream = await spead2.send.asyncio.TcpStream.connect( + thread_pool, args.host, args.port, config, args.buffer, args.bind) elif 'ibv' in args and args.ibv: stream = spead2.send.asyncio.UdpIbvStream( thread_pool, args.host, args.port, config, args.bind, @@ -165,7 +190,7 @@ stream = spead2.send.asyncio.UdpStream( thread_pool, args.host, args.port, config, args.buffer, **kwargs) - yield from(run(item_group, stream, args)) + await run(item_group, stream, args) def main(): @@ -173,6 +198,3 @@ asyncio.get_event_loop().run_until_complete(async_main()) except KeyboardInterrupt: sys.exit(1) -# Monkey-patch asyncio.ensure_future for Python < 3.4.4. -if not hasattr(asyncio, 'ensure_future'): - asyncio.ensure_future = getattr(asyncio, 'async') diff -Nru spead2-1.10.0/spead2/tools/send_trollius.py spead2-2.1.0/spead2/tools/send_trollius.py --- spead2-1.10.0/spead2/tools/send_trollius.py 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/spead2/tools/send_trollius.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,173 +0,0 @@ -# Copyright 2015, 2017 SKA South Africa -# -# This program is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . - -"""Generate and send SPEAD packets. This is mainly a benchmark application, but also -demonstrates the API.""" - -from __future__ import print_function, division -import spead2 -import spead2.send -import spead2.send.trollius -import numpy as np -import logging -import sys -import time -import itertools -import argparse -import trollius -import collections -from trollius import From - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('host') - parser.add_argument('port', type=int) - - group = parser.add_argument_group('Data options') - group.add_argument('--heap-size', metavar='BYTES', type=int, default=4194304, help='Payload size for heap [%(default)s]') - group.add_argument('--items', type=int, default=1, help='Number of items per heap [%(default)s]') - group.add_argument('--dtype', type=str, default='= 2: - try: - n_bytes += yield From(tasks.popleft()) - except Exception as error: - n_errors += 1 - last_error = error - if is_end: - task = trollius.ensure_future(stream.async_send_heap(item_group.get_end())) - else: - for item in item_group.values(): - item.version += 1 - task = trollius.ensure_future(stream.async_send_heap(item_group.get_heap())) - tasks.append(task) - while len(tasks) > 0: - try: - n_bytes += yield From(tasks.popleft()) - except Exception as error: - n_errors += 1 - last_error = error - elapsed = time.time() - start_time - if last_error is not None: - logging.warn('%d errors, last one: %s', n_errors, last_error) - print('Sent {} bytes in {:.6f}s, {:.6f} Gb/s'.format( - n_bytes, elapsed, n_bytes * 8 / elapsed / 1e9)) - - -@trollius.coroutine -def async_main(): - args = get_args() - logging.basicConfig(level=getattr(logging, args.log.upper())) - - dtype = np.dtype(args.dtype) - elements = args.heap_size // (args.items * dtype.itemsize) - heap_size = elements * args.items * dtype.itemsize - if heap_size != args.heap_size: - logging.warn('Heap size is not an exact multiple: using %d instead of %d', - heap_size, args.heap_size) - bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2 if args.pyspead else 0 - item_group = spead2.send.ItemGroup( - descriptor_frequency=args.descriptors, - flavour=spead2.Flavour(4, 64, args.addr_bits, bug_compat)) - for i in range(args.items): - item_group.add_item(id=None, name='Test item {}'.format(i), - description='A test item with arbitrary value', - shape=(elements,), dtype=dtype, - value=np.zeros((elements,), dtype=dtype)) - if args.affinity is not None and len(args.affinity) > 0: - spead2.ThreadPool.set_affinity(args.affinity[0]) - thread_pool = spead2.ThreadPool(args.threads, args.affinity[1:] + args.affinity[:1]) - else: - thread_pool = spead2.ThreadPool(args.threads) - config = spead2.send.StreamConfig( - max_packet_size=args.packet, - burst_size=args.burst, - rate=args.rate * 10**9 / 8, - burst_rate_ratio=args.burst_rate_ratio) - if args.tcp: - stream = yield From(spead2.send.trollius.TcpStream.connect( - thread_pool, args.host, args.port, config, args.buffer, args.bind)) - elif 'ibv' in args and args.ibv: - stream = spead2.send.trollius.UdpIbvStream( - thread_pool, args.host, args.port, config, args.bind, - args.buffer, args.ttl or 1, args.ibv_vector, args.ibv_max_poll) - else: - kwargs = {} - if args.ttl is not None: - kwargs['ttl'] = args.ttl - if args.bind: - kwargs['interface_address'] = args.bind - stream = spead2.send.trollius.UdpStream( - thread_pool, args.host, args.port, config, args.buffer, **kwargs) - - yield From(run(item_group, stream, args)) - - -def main(): - try: - trollius.get_event_loop().run_until_complete(async_main()) - except KeyboardInterrupt: - sys.exit(1) diff -Nru spead2-1.10.0/spead2/_version.py spead2-2.1.0/spead2/_version.py --- spead2-1.10.0/spead2/_version.py 2018-12-11 11:29:32.000000000 +0000 +++ spead2-2.1.0/spead2/_version.py 2020-02-14 08:27:37.000000000 +0000 @@ -1 +1 @@ -__version__ = "1.10.0" +__version__ = "2.1.0" diff -Nru spead2-1.10.0/spead2.egg-info/entry_points.txt spead2-2.1.0/spead2.egg-info/entry_points.txt --- spead2-1.10.0/spead2.egg-info/entry_points.txt 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/spead2.egg-info/entry_points.txt 2020-02-14 08:54:38.000000000 +0000 @@ -0,0 +1,5 @@ +[console_scripts] +spead2_bench.py = spead2.tools.bench_asyncio:main +spead2_recv.py = spead2.tools.recv_asyncio:main +spead2_send.py = spead2.tools.send_asyncio:main + diff -Nru spead2-1.10.0/spead2.egg-info/PKG-INFO spead2-2.1.0/spead2.egg-info/PKG-INFO --- spead2-1.10.0/spead2.egg-info/PKG-INFO 2018-12-11 11:46:36.000000000 +0000 +++ spead2-2.1.0/spead2.egg-info/PKG-INFO 2020-02-14 08:54:38.000000000 +0000 @@ -1,19 +1,54 @@ -Metadata-Version: 1.1 +Metadata-Version: 1.2 Name: spead2 -Version: 1.10.0 +Version: 2.1.0 Summary: High-performance SPEAD implementation Home-page: https://github.com/ska-sa/spead2 Author: Bruce Merry Author-email: bmerry@ska.ac.za License: LGPLv3+ -Description: UNKNOWN +Description: spead2 + ====== + + .. image:: https://readthedocs.org/projects/spead2/badge/?version=latest + :target: https://readthedocs.org/projects/spead2/?badge=latest + :alt: Documentation Status + + .. image:: https://travis-ci.org/ska-sa/spead2.svg?branch=master + :target: https://travis-ci.org/ska-sa/spead2 + :alt: Test Status + + .. image:: https://coveralls.io/repos/github/ska-sa/spead2/badge.svg + :target: https://coveralls.io/github/ska-sa/spead2 + :alt: Coverage Status + + spead2 is an implementation of the SPEAD_ protocol, with both Python and C++ + bindings. The *2* in the name indicates that this is a new implementation of + the protocol; the protocol remains essentially the same. Compared to the + PySPEAD_ implementation, spead2: + + - is at least an order of magnitude faster when dealing with large heaps; + - correctly implements several aspects of the protocol that were implemented + incorrectly in PySPEAD (bug-compatibility is also available); + - correctly implements many corner cases on which PySPEAD would simply fail; + - cleanly supports several SPEAD flavours (e.g. 64-40 and 64-48) in one + module, with the receiver adapting to the flavour used by the sender; + - supports Python 3; + - supports asynchronous operation, using asyncio_. + + For more information, refer to the documentation on readthedocs_. + + .. _SPEAD: https://casper.berkeley.edu/wiki/SPEAD + .. _PySPEAD: https://github.com/ska-sa/PySPEAD/ + .. _asyncio: https://docs.python.org/3/library/asyncio.html + .. _readthedocs: http://spead2.readthedocs.io/en/latest/ + Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Framework :: AsyncIO Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) Classifier: Operating System :: POSIX -Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: System :: Networking +Requires-Python: >=3.5 diff -Nru spead2-1.10.0/spead2.egg-info/requires.txt spead2-2.1.0/spead2.egg-info/requires.txt --- spead2-1.10.0/spead2.egg-info/requires.txt 2018-12-11 11:46:36.000000000 +0000 +++ spead2-2.1.0/spead2.egg-info/requires.txt 2020-02-14 08:54:38.000000000 +0000 @@ -1,5 +1 @@ numpy>=1.9.2 -six - -[:python_version < "3.4"] -trollius diff -Nru spead2-1.10.0/spead2.egg-info/SOURCES.txt spead2-2.1.0/spead2.egg-info/SOURCES.txt --- spead2-1.10.0/spead2.egg-info/SOURCES.txt 2018-12-11 11:46:36.000000000 +0000 +++ spead2-2.1.0/spead2.egg-info/SOURCES.txt 2020-02-14 08:54:39.000000000 +0000 @@ -1,3 +1,4 @@ +.flake8 .gitmodules .readthedocs.yml .travis.yml @@ -106,12 +107,15 @@ 3rdparty/pybind11/tests/CMakeLists.txt 3rdparty/pybind11/tests/conftest.py 3rdparty/pybind11/tests/constructor_stats.h +3rdparty/pybind11/tests/cross_module_gil_utils.cpp 3rdparty/pybind11/tests/local_bindings.h 3rdparty/pybind11/tests/object.h 3rdparty/pybind11/tests/pybind11_cross_module_tests.cpp 3rdparty/pybind11/tests/pybind11_tests.cpp 3rdparty/pybind11/tests/pybind11_tests.h 3rdparty/pybind11/tests/pytest.ini +3rdparty/pybind11/tests/test_async.cpp +3rdparty/pybind11/tests/test_async.py 3rdparty/pybind11/tests/test_buffers.cpp 3rdparty/pybind11/tests/test_buffers.py 3rdparty/pybind11/tests/test_builtin_casters.cpp @@ -141,6 +145,8 @@ 3rdparty/pybind11/tests/test_exceptions.py 3rdparty/pybind11/tests/test_factory_constructors.cpp 3rdparty/pybind11/tests/test_factory_constructors.py +3rdparty/pybind11/tests/test_gil_scoped.cpp +3rdparty/pybind11/tests/test_gil_scoped.py 3rdparty/pybind11/tests/test_iostream.cpp 3rdparty/pybind11/tests/test_iostream.py 3rdparty/pybind11/tests/test_kwargs_and_defaults.cpp @@ -175,6 +181,10 @@ 3rdparty/pybind11/tests/test_stl.py 3rdparty/pybind11/tests/test_stl_binders.cpp 3rdparty/pybind11/tests/test_stl_binders.py +3rdparty/pybind11/tests/test_tagbased_polymorphic.cpp +3rdparty/pybind11/tests/test_tagbased_polymorphic.py +3rdparty/pybind11/tests/test_union.cpp +3rdparty/pybind11/tests/test_union.py 3rdparty/pybind11/tests/test_virtual_functions.cpp 3rdparty/pybind11/tests/test_virtual_functions.py 3rdparty/pybind11/tests/test_cmake_build/CMakeLists.txt @@ -221,7 +231,6 @@ doc/cpp-ibverbs.rst doc/cpp-inproc.rst doc/cpp-logging.rst -doc/cpp-netmap.rst doc/cpp-recv.rst doc/cpp-send.rst doc/cpp-stability.rst @@ -249,15 +258,19 @@ examples/test_ringbuffer.cpp examples/test_send.cpp examples/test_send.py -examples/test_send_trollius.py +examples/test_send_asyncio.py +gen/gen_ibv_loader.py include/Makefile.am include/Makefile.in +include/spead2/.gitignore include/spead2/common_bind.h include/spead2/common_defines.h include/spead2/common_endian.h include/spead2/common_features.h.in include/spead2/common_flavour.h include/spead2/common_ibv.h +include/spead2/common_ibv_loader.h +include/spead2/common_ibv_loader_utils.h include/spead2/common_inproc.h include/spead2/common_logging.h include/spead2/common_memcpy.h @@ -277,7 +290,6 @@ include/spead2/recv_inproc.h include/spead2/recv_live_heap.h include/spead2/recv_mem.h -include/spead2/recv_netmap.h include/spead2/recv_packet.h include/spead2/recv_reader.h include/spead2/recv_ring_stream.h @@ -286,6 +298,7 @@ include/spead2/recv_udp.h include/spead2/recv_udp_base.h include/spead2/recv_udp_ibv.h +include/spead2/recv_udp_ibv_mprq.h include/spead2/recv_udp_pcap.h include/spead2/recv_utils.h include/spead2/send_heap.h @@ -299,50 +312,49 @@ include/spead2/send_utils.h m4/spead2_arg_with.m4 m4/spead2_check_feature.m4 -manylinux1/Dockerfile -manylinux1/generate_wheels.sh -manylinux1/install_requirements.sh -scripts/spead2_bench.py -scripts/spead2_recv.py -scripts/spead2_send.py +manylinux/Dockerfile +manylinux/generate_wheels.sh +manylinux/install_requirements.sh spead2/.gitignore spead2/__init__.py +spead2/__init__.pyi spead2/_version.py +spead2/py.typed spead2.egg-info/PKG-INFO spead2.egg-info/SOURCES.txt spead2.egg-info/dependency_links.txt +spead2.egg-info/entry_points.txt spead2.egg-info/requires.txt spead2.egg-info/top_level.txt spead2/recv/__init__.py +spead2/recv/__init__.pyi spead2/recv/asyncio.py -spead2/recv/trollius.py +spead2/recv/asyncio.pyi spead2/send/__init__.py +spead2/send/__init__.pyi spead2/send/asyncio.py -spead2/send/trollius.py +spead2/send/asyncio.pyi spead2/test/__init__.py spead2/test/shutdown.py spead2/test/test_common.py spead2/test/test_passthrough.py spead2/test/test_passthrough_asyncio.py -spead2/test/test_passthrough_trollius.py spead2/test/test_recv.py -spead2/test/test_recv_asyncio_py35.py +spead2/test/test_recv_asyncio.py spead2/test/test_send.py spead2/test/test_send_asyncio.py -spead2/test/test_send_trollius.py spead2/tools/__init__.py spead2/tools/bench_asyncio.py -spead2/tools/bench_trollius.py spead2/tools/recv_asyncio.py -spead2/tools/recv_trollius.py spead2/tools/send_asyncio.py -spead2/tools/send_trollius.py src/.gitignore src/Makefile.am src/Makefile.in src/Makefile.inc.am src/common_flavour.cpp src/common_ibv.cpp +src/common_ibv_loader.cpp +src/common_ibv_loader_utils.cpp src/common_inproc.cpp src/common_logging.cpp src/common_memcpy.cpp @@ -361,7 +373,6 @@ src/recv_inproc.cpp src/recv_live_heap.cpp src/recv_mem.cpp -src/recv_netmap.cpp src/recv_packet.cpp src/recv_reader.cpp src/recv_ring_stream.cpp @@ -370,6 +381,7 @@ src/recv_udp.cpp src/recv_udp_base.cpp src/recv_udp_ibv.cpp +src/recv_udp_ibv_mprq.cpp src/recv_udp_pcap.cpp src/send_heap.cpp src/send_inproc.cpp @@ -382,9 +394,14 @@ src/spead2_bench.cpp src/spead2_recv.cpp src/spead2_send.cpp +src/unittest_logging.cpp src/unittest_main.cpp src/unittest_memcpy.cpp src/unittest_memory_allocator.cpp src/unittest_memory_pool.cpp +src/unittest_raw_packet.cpp +src/unittest_recv_custom_memcpy.cpp src/unittest_recv_live_heap.cpp -src/unittest_send_heap.cpp \ No newline at end of file +src/unittest_semaphore.cpp +src/unittest_send_heap.cpp +src/unittest_send_streambuf.cpp \ No newline at end of file diff -Nru spead2-1.10.0/src/common_ibv.cpp spead2-2.1.0/src/common_ibv.cpp --- spead2-1.10.0/src/common_ibv.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/common_ibv.cpp 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016, 2017 SKA South Africa +/* Copyright 2016-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,18 +24,53 @@ #include #include #include +#include +#include #include #include #include #include +#include +#include #include #include namespace spead2 { +namespace detail +{ + +#if SPEAD2_USE_IBV_MPRQ +ibv_intf_deleter::ibv_intf_deleter(struct ibv_context *context) noexcept : context(context) {} + +void ibv_intf_deleter::operator()(void *intf) +{ + assert(context); + struct ibv_exp_release_intf_params params; + std::memset(¶ms, 0, sizeof(params)); + ibv_exp_release_intf(context, intf, ¶ms); +} + +ibv_exp_res_domain_deleter::ibv_exp_res_domain_deleter(struct ibv_context *context) noexcept + : context(context) +{ +} + +void ibv_exp_res_domain_deleter::operator()(ibv_exp_res_domain *res_domain) +{ + ibv_exp_destroy_res_domain_attr attr; + std::memset(&attr, 0, sizeof(attr)); + ibv_exp_destroy_res_domain(context, res_domain, &attr); +} + +#endif + +} // namespace detail + rdma_event_channel_t::rdma_event_channel_t() { + ibv_loader_init(); errno = 0; rdma_event_channel *event_channel = rdma_create_event_channel(); if (!event_channel) @@ -45,6 +80,7 @@ rdma_cm_id_t::rdma_cm_id_t(const rdma_event_channel_t &event_channel, void *context, rdma_port_space ps) { + ibv_loader_init(); rdma_cm_id *cm_id = nullptr; errno = 0; int status = rdma_create_id(event_channel.get(), &cm_id, context, ps); @@ -65,6 +101,77 @@ throw_errno("rdma_bind_addr did not bind to an RDMA device", ENODEV); } +ibv_device_attr rdma_cm_id_t::query_device() const +{ + assert(get()); + ibv_device_attr attr; + std::memset(&attr, 0, sizeof(attr)); + int status = ibv_query_device(get()->verbs, &attr); + if (status != 0) + throw_errno("ibv_query_device failed", status); + return attr; +} + +#if SPEAD2_USE_IBV_EXP +ibv_exp_device_attr rdma_cm_id_t::exp_query_device() const +{ + assert(get()); + ibv_exp_device_attr attr; + std::memset(&attr, 0, sizeof(attr)); + attr.comp_mask = IBV_EXP_DEVICE_ATTR_RESERVED - 1; + int status = ibv_exp_query_device(get()->verbs, &attr); + if (status != 0) + throw_errno("ibv_exp_query_device failed", status); + return attr; +} +#endif + +ibv_context_t::ibv_context_t(struct ibv_device *device) +{ + ibv_context *ctx = ibv_open_device(device); + if (!ctx) + throw_errno("ibv_open_device failed"); + reset(ctx); +} + +ibv_context_t::ibv_context_t(const boost::asio::ip::address &addr) +{ + /* Use rdma_cm_id_t to get an existing device context, then + * query it for its GUID and find the corresponding device. + */ + rdma_event_channel_t event_channel; + rdma_cm_id_t cm_id(event_channel, nullptr, RDMA_PS_UDP); + cm_id.bind_addr(addr); + ibv_device_attr attr = cm_id.query_device(); + + struct ibv_device **devices; + devices = ibv_get_device_list(nullptr); + if (devices == nullptr) + throw_errno("ibv_get_device_list failed"); + + ibv_device *device = nullptr; + for (ibv_device **d = devices; *d != nullptr; d++) + if (ibv_get_device_guid(*d) == attr.node_guid) + { + device = *d; + break; + } + if (device == nullptr) + { + ibv_free_device_list(devices); + throw_errno("no matching device found", ENOENT); + } + + ibv_context *ctx = ibv_open_device(device); + if (!ctx) + { + ibv_free_device_list(devices); + throw_errno("ibv_open_device failed"); + } + reset(ctx); + ibv_free_device_list(devices); +} + ibv_comp_channel_t::ibv_comp_channel_t(const rdma_cm_id_t &cm_id) { errno = 0; @@ -81,13 +188,19 @@ return wrap_fd(io_service, get()->fd); } -void ibv_comp_channel_t::get_event(ibv_cq **cq, void **context) +bool ibv_comp_channel_t::get_event(ibv_cq **cq, void **context) { assert(get()); errno = 0; int status = ibv_get_cq_event(get(), cq, context); if (status < 0) - throw_errno("ibv_get_cq_event failed"); + { + if (errno == EAGAIN) + return false; + else + throw_errno("ibv_get_cq_event failed"); + } + return true; } ibv_cq_t::ibv_cq_t( @@ -187,6 +300,17 @@ reset(qp); } +#if SPEAD2_USE_IBV_MPRQ +ibv_qp_t::ibv_qp_t(const rdma_cm_id_t &cm_id, ibv_exp_qp_init_attr *init_attr) +{ + errno = 0; + ibv_qp *qp = ibv_exp_create_qp(cm_id->verbs, init_attr); + if (!qp) + throw_errno("ibv_exp_create_qp failed"); + reset(qp); +} +#endif + ibv_mr_t::ibv_mr_t(const ibv_pd_t &pd, void *addr, std::size_t length, int access) { errno = 0; @@ -248,6 +372,224 @@ reset(flow); } +ibv_flow_t create_flow( + const ibv_qp_t &qp, const boost::asio::ip::udp::endpoint &endpoint, + int port_num) +{ + struct + { + ibv_flow_attr attr; + ibv_flow_spec_eth eth __attribute__((packed)); + ibv_flow_spec_ipv4 ip __attribute__((packed)); + ibv_flow_spec_tcp_udp udp __attribute__((packed)); + } flow_rule; + std::memset(&flow_rule, 0, sizeof(flow_rule)); + + flow_rule.attr.type = IBV_FLOW_ATTR_NORMAL; + flow_rule.attr.priority = 0; + flow_rule.attr.size = sizeof(flow_rule); + flow_rule.attr.num_of_specs = 3; + flow_rule.attr.port = port_num; + + flow_rule.eth.type = IBV_FLOW_SPEC_ETH; + flow_rule.eth.size = sizeof(flow_rule.eth); + flow_rule.ip.type = IBV_FLOW_SPEC_IPV4; + flow_rule.ip.size = sizeof(flow_rule.ip); + + if (!endpoint.address().is_unspecified()) + { + /* At least the ConnectX-3 cards seem to require an Ethernet match. We + * thus have to construct the either the MAC address corresponding to + * the IP multicast address from RFC 7042, the interface address for + * unicast. + */ + mac_address dst_mac; + if (endpoint.address().is_multicast()) + dst_mac = multicast_mac(endpoint.address()); + else + dst_mac = interface_mac(endpoint.address()); + std::memcpy(&flow_rule.eth.val.dst_mac, &dst_mac, sizeof(dst_mac)); + std::memset(&flow_rule.eth.mask.dst_mac, 0xFF, sizeof(flow_rule.eth.mask.dst_mac)); + + auto bytes = endpoint.address().to_v4().to_bytes(); // big-endian address + std::memcpy(&flow_rule.ip.val.dst_ip, &bytes, sizeof(bytes)); + std::memset(&flow_rule.ip.mask.dst_ip, 0xFF, sizeof(flow_rule.ip.mask.dst_ip)); + } + + flow_rule.udp.type = IBV_FLOW_SPEC_UDP; + flow_rule.udp.size = sizeof(flow_rule.udp); + flow_rule.udp.val.dst_port = htobe16(endpoint.port()); + flow_rule.udp.mask.dst_port = 0xFFFF; + + return ibv_flow_t(qp, &flow_rule.attr); +} + +std::vector create_flows( + const ibv_qp_t &qp, + const std::vector &endpoints, + int port_num) +{ + /* Note: some NICs support flow rules with non-trivial masks. However, + * using such rules can lead to subtle problems when there are multiple + * receivers on the same NIC subscribing to common groups. See #66 for + * more details. + */ + std::vector flows; + for (const auto &ep : endpoints) + flows.push_back(create_flow(qp, ep, port_num)); + return flows; +} + +#if SPEAD2_USE_IBV_MPRQ + +const char *ibv_exp_query_intf_error_category::name() const noexcept +{ + return "ibv_exp_query_intf"; +} + +std::string ibv_exp_query_intf_error_category::message(int condition) const +{ + switch (condition) + { + case IBV_EXP_INTF_STAT_OK: + return "OK"; + case IBV_EXP_INTF_STAT_VENDOR_NOT_SUPPORTED: + return "The provided 'vendor_guid' is not supported"; + case IBV_EXP_INTF_STAT_INTF_NOT_SUPPORTED: + return "The provided 'intf' is not supported"; + case IBV_EXP_INTF_STAT_VERSION_NOT_SUPPORTED: + return "The provided 'intf_version' is not supported"; + case IBV_EXP_INTF_STAT_INVAL_PARARM: + return "General invalid parameter"; + case IBV_EXP_INTF_STAT_INVAL_OBJ_STATE: + return "QP is not in INIT, RTR or RTS state"; + case IBV_EXP_INTF_STAT_INVAL_OBJ: + return "Mismatch between the provided 'obj'(CQ/QP/WQ) and requested 'intf'"; + case IBV_EXP_INTF_STAT_FLAGS_NOT_SUPPORTED: + return "The provided set of 'flags' is not supported"; + case IBV_EXP_INTF_STAT_FAMILY_FLAGS_NOT_SUPPORTED: + return "The provided set of 'family_flags' is not supported"; + default: + return "Unknown error"; + } +} + +std::error_condition ibv_exp_query_intf_error_category::default_error_condition(int condition) const noexcept +{ + switch (condition) + { + case IBV_EXP_INTF_STAT_VENDOR_NOT_SUPPORTED: + case IBV_EXP_INTF_STAT_INTF_NOT_SUPPORTED: + case IBV_EXP_INTF_STAT_VERSION_NOT_SUPPORTED: + case IBV_EXP_INTF_STAT_FLAGS_NOT_SUPPORTED: + case IBV_EXP_INTF_STAT_FAMILY_FLAGS_NOT_SUPPORTED: + return std::errc::not_supported; + case IBV_EXP_INTF_STAT_INVAL_PARARM: + case IBV_EXP_INTF_STAT_INVAL_OBJ_STATE: + case IBV_EXP_INTF_STAT_INVAL_OBJ: + return std::errc::invalid_argument; + default: + return std::error_condition(condition, *this); + } +} + +std::error_category &ibv_exp_query_intf_category() +{ + static ibv_exp_query_intf_error_category category; + return category; +} + +static void *query_intf(const rdma_cm_id_t &cm_id, ibv_exp_query_intf_params *params) +{ + ibv_exp_query_intf_status status; + void *intf = ibv_exp_query_intf(cm_id->verbs, params, &status); + if (status != IBV_EXP_INTF_STAT_OK) + { + std::error_code code(status, ibv_exp_query_intf_category()); + throw std::system_error(code, "ibv_exp_query_intf failed"); + } + return intf; +} + +ibv_exp_cq_family_v1_t::ibv_exp_cq_family_v1_t(const rdma_cm_id_t &cm_id, const ibv_cq_t &cq) + : std::unique_ptr( + nullptr, detail::ibv_intf_deleter(cm_id->verbs)) +{ + ibv_exp_query_intf_params params; + std::memset(¶ms, 0, sizeof(params)); + params.intf_scope = IBV_EXP_INTF_GLOBAL; + params.intf = IBV_EXP_INTF_CQ; + params.intf_version = 1; + params.obj = cq.get(); + void *intf = query_intf(cm_id, ¶ms); + reset(static_cast(intf)); +} + +ibv_exp_wq_t::ibv_exp_wq_t(const rdma_cm_id_t &cm_id, ibv_exp_wq_init_attr *attr) +{ + ibv_exp_wq *wq = ibv_exp_create_wq(cm_id->verbs, attr); + if (!wq) + throw_errno("ibv_exp_create_wq failed"); + reset(wq); +} + +void ibv_exp_wq_t::modify(ibv_exp_wq_state state) +{ + ibv_exp_wq_attr wq_attr; + std::memset(&wq_attr, 0, sizeof(wq_attr)); + wq_attr.wq_state = IBV_EXP_WQS_RDY; + wq_attr.attr_mask = IBV_EXP_WQ_ATTR_STATE; + int status = ibv_exp_modify_wq(get(), &wq_attr); + if (status != 0) + throw_errno("ibv_exp_modify_wq failed", status); +} + +ibv_exp_wq_family_t::ibv_exp_wq_family_t(const rdma_cm_id_t &cm_id, const ibv_exp_wq_t &wq) + : std::unique_ptr( + nullptr, detail::ibv_intf_deleter(cm_id->verbs)) +{ + ibv_exp_query_intf_params params; + std::memset(¶ms, 0, sizeof(params)); + params.intf_scope = IBV_EXP_INTF_GLOBAL; + params.intf = IBV_EXP_INTF_WQ; + params.obj = wq.get(); + void *intf = query_intf(cm_id, ¶ms); + reset(static_cast(intf)); +} + +ibv_exp_rwq_ind_table_t::ibv_exp_rwq_ind_table_t(const rdma_cm_id_t &cm_id, ibv_exp_rwq_ind_table_init_attr *attr) +{ + ibv_exp_rwq_ind_table *table = ibv_exp_create_rwq_ind_table(cm_id->verbs, attr); + if (!table) + throw_errno("ibv_exp_create_rwq_ind_table failed"); + reset(table); +} + +ibv_exp_rwq_ind_table_t create_rwq_ind_table( + const rdma_cm_id_t &cm_id, const ibv_pd_t &pd, const ibv_exp_wq_t &wq) +{ + ibv_exp_rwq_ind_table_init_attr attr; + ibv_exp_wq *tbl[1] = {wq.get()}; + std::memset(&attr, 0, sizeof(attr)); + attr.pd = pd.get(); + attr.log_ind_tbl_size = 0; + attr.ind_tbl = tbl; + return ibv_exp_rwq_ind_table_t(cm_id, &attr); +} + +ibv_exp_res_domain_t::ibv_exp_res_domain_t(const rdma_cm_id_t &cm_id, ibv_exp_res_domain_init_attr *attr) + : std::unique_ptr( + nullptr, detail::ibv_exp_res_domain_deleter(cm_id->verbs)) +{ + errno = 0; + ibv_exp_res_domain *res_domain = ibv_exp_create_res_domain(cm_id->verbs, attr); + if (!res_domain) + throw_errno("ibv_exp_create_res_domain_failed"); + reset(res_domain); +} + +#endif // SPEAD2_USE_IBV_MPRQ + } // namespace spead #endif // SPEAD2_USE_IBV diff -Nru spead2-1.10.0/src/common_ibv_loader.cpp spead2-2.1.0/src/common_ibv_loader.cpp --- spead2-1.10.0/src/common_ibv_loader.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/common_ibv_loader.cpp 2020-02-14 08:26:56.000000000 +0000 @@ -0,0 +1,294 @@ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include +#include +#include +#include + +namespace spead2 +{ + +static std::once_flag init_once; +static std::exception_ptr init_result; + +void ibv_ack_cq_events_stub(struct ibv_cq *cq, unsigned int nevents) +{ + (void) cq; + (void) nevents; + ibv_loader_stub(init_result); +} +struct ibv_pd *ibv_alloc_pd_stub(struct ibv_context *context) +{ + (void) context; + ibv_loader_stub(init_result); +} +int ibv_close_device_stub(struct ibv_context *context) +{ + (void) context; + ibv_loader_stub(init_result); +} +struct ibv_comp_channel *ibv_create_comp_channel_stub(struct ibv_context *context) +{ + (void) context; + ibv_loader_stub(init_result); +} +struct ibv_cq *ibv_create_cq_stub(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector) +{ + (void) context; + (void) cqe; + (void) cq_context; + (void) channel; + (void) comp_vector; + ibv_loader_stub(init_result); +} +struct ibv_qp *ibv_create_qp_stub(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) +{ + (void) pd; + (void) qp_init_attr; + ibv_loader_stub(init_result); +} +int ibv_dealloc_pd_stub(struct ibv_pd *pd) +{ + (void) pd; + ibv_loader_stub(init_result); +} +int ibv_dereg_mr_stub(struct ibv_mr *mr) +{ + (void) mr; + ibv_loader_stub(init_result); +} +int ibv_destroy_comp_channel_stub(struct ibv_comp_channel *channel) +{ + (void) channel; + ibv_loader_stub(init_result); +} +int ibv_destroy_cq_stub(struct ibv_cq *cq) +{ + (void) cq; + ibv_loader_stub(init_result); +} +int ibv_destroy_qp_stub(struct ibv_qp *qp) +{ + (void) qp; + ibv_loader_stub(init_result); +} +void ibv_free_device_list_stub(struct ibv_device **list) +{ + (void) list; + ibv_loader_stub(init_result); +} +int ibv_get_cq_event_stub(struct ibv_comp_channel *channel, struct ibv_cq **cq, void **cq_context) +{ + (void) channel; + (void) cq; + (void) cq_context; + ibv_loader_stub(init_result); +} +uint64_t ibv_get_device_guid_stub(struct ibv_device *device) +{ + (void) device; + ibv_loader_stub(init_result); +} +struct ibv_device **ibv_get_device_list_stub(int *num_devices) +{ + (void) num_devices; + ibv_loader_stub(init_result); +} +struct ibv_context *ibv_open_device_stub(struct ibv_device *device) +{ + (void) device; + ibv_loader_stub(init_result); +} +int ibv_modify_qp_stub(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) +{ + (void) qp; + (void) attr; + (void) attr_mask; + ibv_loader_stub(init_result); +} +int ibv_query_device_stub(struct ibv_context *context, struct ibv_device_attr *device_attr) +{ + (void) context; + (void) device_attr; + ibv_loader_stub(init_result); +} +struct ibv_mr *ibv_reg_mr_stub(struct ibv_pd *pd, void *addr, size_t length, int access) +{ + (void) pd; + (void) addr; + (void) length; + (void) access; + ibv_loader_stub(init_result); +} +int rdma_bind_addr_stub(struct rdma_cm_id *id, struct sockaddr *addr) +{ + (void) id; + (void) addr; + ibv_loader_stub(init_result); +} +struct rdma_event_channel *rdma_create_event_channel_stub(void) +{ + ibv_loader_stub(init_result); +} +int rdma_create_id_stub(struct rdma_event_channel *channel, struct rdma_cm_id **id, void *context, enum rdma_port_space ps) +{ + (void) channel; + (void) id; + (void) context; + (void) ps; + ibv_loader_stub(init_result); +} +void rdma_destroy_event_channel_stub(struct rdma_event_channel *channel) +{ + (void) channel; + ibv_loader_stub(init_result); +} +int rdma_destroy_id_stub(struct rdma_cm_id *id) +{ + (void) id; + ibv_loader_stub(init_result); +} + +void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents) = ibv_ack_cq_events_stub; +struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context) = ibv_alloc_pd_stub; +int (*ibv_close_device)(struct ibv_context *context) = ibv_close_device_stub; +struct ibv_comp_channel *(*ibv_create_comp_channel)(struct ibv_context *context) = ibv_create_comp_channel_stub; +struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector) = ibv_create_cq_stub; +struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) = ibv_create_qp_stub; +int (*ibv_dealloc_pd)(struct ibv_pd *pd) = ibv_dealloc_pd_stub; +int (*ibv_dereg_mr)(struct ibv_mr *mr) = ibv_dereg_mr_stub; +int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel) = ibv_destroy_comp_channel_stub; +int (*ibv_destroy_cq)(struct ibv_cq *cq) = ibv_destroy_cq_stub; +int (*ibv_destroy_qp)(struct ibv_qp *qp) = ibv_destroy_qp_stub; +void (*ibv_free_device_list)(struct ibv_device **list) = ibv_free_device_list_stub; +int (*ibv_get_cq_event)(struct ibv_comp_channel *channel, struct ibv_cq **cq, void **cq_context) = ibv_get_cq_event_stub; +uint64_t (*ibv_get_device_guid)(struct ibv_device *device) = ibv_get_device_guid_stub; +struct ibv_device **(*ibv_get_device_list)(int *num_devices) = ibv_get_device_list_stub; +struct ibv_context *(*ibv_open_device)(struct ibv_device *device) = ibv_open_device_stub; +int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) = ibv_modify_qp_stub; +int (*ibv_query_device)(struct ibv_context *context, struct ibv_device_attr *device_attr) = ibv_query_device_stub; +struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr, size_t length, int access) = ibv_reg_mr_stub; +int (*rdma_bind_addr)(struct rdma_cm_id *id, struct sockaddr *addr) = rdma_bind_addr_stub; +struct rdma_event_channel *(*rdma_create_event_channel)(void) = rdma_create_event_channel_stub; +int (*rdma_create_id)(struct rdma_event_channel *channel, struct rdma_cm_id **id, void *context, enum rdma_port_space ps) = rdma_create_id_stub; +void (*rdma_destroy_event_channel)(struct rdma_event_channel *channel) = rdma_destroy_event_channel_stub; +int (*rdma_destroy_id)(struct rdma_cm_id *id) = rdma_destroy_id_stub; + +static void reset_stubs() +{ + ibv_ack_cq_events = ibv_ack_cq_events_stub; + ibv_alloc_pd = ibv_alloc_pd_stub; + ibv_close_device = ibv_close_device_stub; + ibv_create_comp_channel = ibv_create_comp_channel_stub; + ibv_create_cq = ibv_create_cq_stub; + ibv_create_qp = ibv_create_qp_stub; + ibv_dealloc_pd = ibv_dealloc_pd_stub; + ibv_dereg_mr = ibv_dereg_mr_stub; + ibv_destroy_comp_channel = ibv_destroy_comp_channel_stub; + ibv_destroy_cq = ibv_destroy_cq_stub; + ibv_destroy_qp = ibv_destroy_qp_stub; + ibv_free_device_list = ibv_free_device_list_stub; + ibv_get_cq_event = ibv_get_cq_event_stub; + ibv_get_device_guid = ibv_get_device_guid_stub; + ibv_get_device_list = ibv_get_device_list_stub; + ibv_open_device = ibv_open_device_stub; + ibv_modify_qp = ibv_modify_qp_stub; + ibv_query_device = ibv_query_device_stub; + ibv_reg_mr = ibv_reg_mr_stub; + rdma_bind_addr = rdma_bind_addr_stub; + rdma_create_event_channel = rdma_create_event_channel_stub; + rdma_create_id = rdma_create_id_stub; + rdma_destroy_event_channel = rdma_destroy_event_channel_stub; + rdma_destroy_id = rdma_destroy_id_stub; +} + +static void init() +{ + try + { + dl_handle librdmacm("librdmacm.so.1"); + dl_handle libibverbs("libibverbs.so.1"); + ibv_ack_cq_events = reinterpret_cast( + libibverbs.sym("ibv_ack_cq_events")); + ibv_alloc_pd = reinterpret_cast( + libibverbs.sym("ibv_alloc_pd")); + ibv_close_device = reinterpret_cast( + libibverbs.sym("ibv_close_device")); + ibv_create_comp_channel = reinterpret_cast( + libibverbs.sym("ibv_create_comp_channel")); + ibv_create_cq = reinterpret_cast( + libibverbs.sym("ibv_create_cq")); + ibv_create_qp = reinterpret_cast( + libibverbs.sym("ibv_create_qp")); + ibv_dealloc_pd = reinterpret_cast( + libibverbs.sym("ibv_dealloc_pd")); + ibv_dereg_mr = reinterpret_cast( + libibverbs.sym("ibv_dereg_mr")); + ibv_destroy_comp_channel = reinterpret_cast( + libibverbs.sym("ibv_destroy_comp_channel")); + ibv_destroy_cq = reinterpret_cast( + libibverbs.sym("ibv_destroy_cq")); + ibv_destroy_qp = reinterpret_cast( + libibverbs.sym("ibv_destroy_qp")); + ibv_free_device_list = reinterpret_cast( + libibverbs.sym("ibv_free_device_list")); + ibv_get_cq_event = reinterpret_cast( + libibverbs.sym("ibv_get_cq_event")); + ibv_get_device_guid = reinterpret_cast( + libibverbs.sym("ibv_get_device_guid")); + ibv_get_device_list = reinterpret_cast( + libibverbs.sym("ibv_get_device_list")); + ibv_open_device = reinterpret_cast( + libibverbs.sym("ibv_open_device")); + ibv_modify_qp = reinterpret_cast( + libibverbs.sym("ibv_modify_qp")); + ibv_query_device = reinterpret_cast( + libibverbs.sym("ibv_query_device")); + ibv_reg_mr = reinterpret_cast( + libibverbs.sym("ibv_reg_mr")); + rdma_bind_addr = reinterpret_cast( + librdmacm.sym("rdma_bind_addr")); + rdma_create_event_channel = reinterpret_cast( + librdmacm.sym("rdma_create_event_channel")); + rdma_create_id = reinterpret_cast( + librdmacm.sym("rdma_create_id")); + rdma_destroy_event_channel = reinterpret_cast( + librdmacm.sym("rdma_destroy_event_channel")); + rdma_destroy_id = reinterpret_cast( + librdmacm.sym("rdma_destroy_id")); + // Prevent the libraries being closed, so that the symbols stay valid + librdmacm.release(); + libibverbs.release(); + } + catch (std::system_error &e) + { + init_result = std::current_exception(); + reset_stubs(); + log_warning("could not load ibverbs: %s", e.what()); + } +} + +void ibv_loader_init() +{ + std::call_once(init_once, init); + if (init_result) + std::rethrow_exception(init_result); +} + +} // namespace spead2 + +/* Wrappers in the global namespace. This is needed because ibv_exp_create_qp + * calls ibv_create_qp, and so we need to provide an implementation. + */ +struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) +{ + return spead2::ibv_create_qp(pd, qp_init_attr); +} + +#endif // SPEAD2_USE_IBV diff -Nru spead2-1.10.0/src/common_ibv_loader_utils.cpp spead2-2.1.0/src/common_ibv_loader_utils.cpp --- spead2-1.10.0/src/common_ibv_loader_utils.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/common_ibv_loader_utils.cpp 2019-05-03 07:50:32.000000000 +0000 @@ -0,0 +1,130 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + */ + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +#if SPEAD2_USE_IBV + +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ + +const char *ibv_loader_error_category::name() const noexcept +{ + return "ibv_loader"; +} + +std::string ibv_loader_error_category::message(int condition) const +{ + switch (ibv_loader_error(condition)) + { + case ibv_loader_error::LIBRARY_ERROR: + return "library could not be loaded"; + case ibv_loader_error::SYMBOL_ERROR: + return "symbol could not be loaded"; + case ibv_loader_error::NO_INIT: + return "ibv_loader_init was not called"; + default: + return "unknown error"; // unreachable + } +} + +std::error_condition ibv_loader_error_category::default_error_condition(int condition) const noexcept +{ + switch (ibv_loader_error(condition)) + { + case ibv_loader_error::LIBRARY_ERROR: + return std::errc::no_such_file_or_directory; + case ibv_loader_error::SYMBOL_ERROR: + return std::errc::not_supported; + case ibv_loader_error::NO_INIT: + return std::errc::state_not_recoverable; + default: + return std::errc::not_supported; // unreachable + } +} + +std::error_category &ibv_loader_category() +{ + static ibv_loader_error_category category; + return category; +} + +dl_handle::dl_handle(const char *filename) +{ + handle = dlopen(filename, RTLD_NOW | RTLD_LOCAL); + if (!handle) + { + std::error_code code((int) ibv_loader_error::LIBRARY_ERROR, ibv_loader_category()); + throw std::system_error(code, std::string("Could not open ") + filename + ": " + dlerror()); + } +} + +dl_handle::~dl_handle() +{ + if (handle) + { + int ret = dlclose(handle); + if (ret != 0) + log_warning("dlclose failed: %s", dlerror()); + } +} + +void *dl_handle::release() +{ + void *ret = handle; + handle = nullptr; + return ret; +} + +void *dl_handle::sym(const char *name) +{ + void *ret = dlsym(handle, name); + if (!ret) + { + std::error_code code((int) ibv_loader_error::SYMBOL_ERROR, ibv_loader_category()); + throw std::system_error(code, std::string("Symbol ") + name + " not found: " + dlerror()); + } + return ret; +} + +void ibv_loader_stub(std::exception_ptr init_result) +{ + if (init_result) + std::rethrow_exception(init_result); + else + { + std::error_code code((int) ibv_loader_error::NO_INIT, ibv_loader_category()); + throw std::system_error(code, "ibv_loader_init was not called"); + } +} + +} // namespace spead2 + +#endif // SPEAD2_USE_IBV diff -Nru spead2-1.10.0/src/common_logging.cpp spead2-2.1.0/src/common_logging.cpp --- spead2-1.10.0/src/common_logging.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/common_logging.cpp 2019-05-14 09:10:08.000000000 +0000 @@ -36,11 +36,16 @@ "debug" }; -static void default_log_function(log_level level, const std::string &msg) +std::ostream &operator<<(std::ostream &o, log_level level) { unsigned int level_idx = static_cast(level); assert(level_idx < sizeof(level_names) / sizeof(level_names[0])); - std::cerr << "spead2: " << level_names[level_idx] << ": " << msg << "\n"; + return o << level_names[level_idx]; +} + +static void default_log_function(log_level level, const std::string &msg) +{ + std::cerr << "spead2: " << level << ": " << msg << "\n"; } static std::function log_function = default_log_function; diff -Nru spead2-1.10.0/src/common_thread_pool.cpp spead2-2.1.0/src/common_thread_pool.cpp --- spead2-1.10.0/src/common_thread_pool.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/common_thread_pool.cpp 2019-10-15 14:42:17.000000000 +0000 @@ -33,6 +33,24 @@ namespace spead2 { +static void run_io_service(boost::asio::io_service &io_service) +{ + try + { + io_service.run(); + } + catch (const std::exception &e) + { + log_warning("Worker thread threw exception (expect deadlocks!): %1%", e.what()); + throw; + } + catch (...) + { + log_warning("Worker thread threw unknown exception (expect deadlocks!)"); + throw; + } +} + thread_pool::thread_pool(int num_threads) : work(io_service) { @@ -41,7 +59,7 @@ workers.reserve(num_threads); for (int i = 0; i < num_threads; i++) { - workers.push_back(std::async(std::launch::async, [this] { io_service.run(); })); + workers.push_back(std::async(std::launch::async, [this] { run_io_service(io_service); })); } } @@ -54,12 +72,14 @@ for (int i = 0; i < num_threads; i++) { if (affinity.empty()) - workers.push_back(std::async(std::launch::async, [this] { io_service.run(); })); + workers.push_back(std::async(std::launch::async, [this] { run_io_service(io_service); })); else { int core = affinity[i % affinity.size()]; workers.push_back(std::async(std::launch::async, [this, core] { - set_affinity(core); io_service.run(); })); + set_affinity(core); + run_io_service(io_service); + })); } } } diff -Nru spead2-1.10.0/src/.gitignore spead2-2.1.0/src/.gitignore --- spead2-1.10.0/src/.gitignore 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/.gitignore 2019-05-03 07:50:32.000000000 +0000 @@ -1,4 +1,5 @@ *.o +common_ibv_loader.cpp spead2_bench spead2_recv spead2_send diff -Nru spead2-1.10.0/src/Makefile.am spead2-2.1.0/src/Makefile.am --- spead2-1.10.0/src/Makefile.am 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/Makefile.am 2019-10-08 08:00:52.000000000 +0000 @@ -37,17 +37,24 @@ spead2_unittest_SOURCES = \ unittest_main.cpp \ + unittest_logging.cpp \ unittest_memcpy.cpp \ unittest_memory_allocator.cpp \ unittest_memory_pool.cpp \ + unittest_raw_packet.cpp \ unittest_recv_live_heap.cpp \ - unittest_send_heap.cpp + unittest_recv_custom_memcpy.cpp \ + unittest_semaphore.cpp \ + unittest_send_heap.cpp \ + unittest_send_streambuf.cpp spead2_unittest_CPPFLAGS = -DBOOST_TEST_DYN_LINK $(AM_CPPFLAGS) spead2_unittest_LDADD = -lboost_unit_test_framework $(LDADD) libspead2_a_SOURCES = \ common_flavour.cpp \ common_ibv.cpp \ + common_ibv_loader.cpp \ + common_ibv_loader_utils.cpp \ common_inproc.cpp \ common_logging.cpp \ common_memcpy.cpp \ @@ -61,7 +68,6 @@ recv_inproc.cpp \ recv_live_heap.cpp \ recv_mem.cpp \ - recv_netmap.cpp \ recv_packet.cpp \ recv_reader.cpp \ recv_ring_stream.cpp \ @@ -70,6 +76,7 @@ recv_udp_base.cpp \ recv_udp.cpp \ recv_udp_ibv.cpp \ + recv_udp_ibv_mprq.cpp \ recv_udp_pcap.cpp \ send_heap.cpp \ send_inproc.cpp \ diff -Nru spead2-1.10.0/src/Makefile.in spead2-2.1.0/src/Makefile.in --- spead2-1.10.0/src/Makefile.in 2018-12-11 11:29:16.000000000 +0000 +++ spead2-2.1.0/src/Makefile.in 2020-02-14 08:26:58.000000000 +0000 @@ -166,21 +166,21 @@ libspead2_a_AR = $(AR) $(ARFLAGS) libspead2_a_LIBADD = am_libspead2_a_OBJECTS = common_flavour.$(OBJEXT) common_ibv.$(OBJEXT) \ + common_ibv_loader.$(OBJEXT) common_ibv_loader_utils.$(OBJEXT) \ common_inproc.$(OBJEXT) common_logging.$(OBJEXT) \ common_memcpy.$(OBJEXT) common_memory_allocator.$(OBJEXT) \ common_memory_pool.$(OBJEXT) common_raw_packet.$(OBJEXT) \ common_semaphore.$(OBJEXT) common_socket.$(OBJEXT) \ common_thread_pool.$(OBJEXT) recv_heap.$(OBJEXT) \ recv_inproc.$(OBJEXT) recv_live_heap.$(OBJEXT) \ - recv_mem.$(OBJEXT) recv_netmap.$(OBJEXT) recv_packet.$(OBJEXT) \ - recv_reader.$(OBJEXT) recv_ring_stream.$(OBJEXT) \ - recv_stream.$(OBJEXT) recv_tcp.$(OBJEXT) \ - recv_udp_base.$(OBJEXT) recv_udp.$(OBJEXT) \ - recv_udp_ibv.$(OBJEXT) recv_udp_pcap.$(OBJEXT) \ - send_heap.$(OBJEXT) send_inproc.$(OBJEXT) \ - send_packet.$(OBJEXT) send_streambuf.$(OBJEXT) \ - send_stream.$(OBJEXT) send_tcp.$(OBJEXT) send_udp.$(OBJEXT) \ - send_udp_ibv.$(OBJEXT) + recv_mem.$(OBJEXT) recv_packet.$(OBJEXT) recv_reader.$(OBJEXT) \ + recv_ring_stream.$(OBJEXT) recv_stream.$(OBJEXT) \ + recv_tcp.$(OBJEXT) recv_udp_base.$(OBJEXT) recv_udp.$(OBJEXT) \ + recv_udp_ibv.$(OBJEXT) recv_udp_ibv_mprq.$(OBJEXT) \ + recv_udp_pcap.$(OBJEXT) send_heap.$(OBJEXT) \ + send_inproc.$(OBJEXT) send_packet.$(OBJEXT) \ + send_streambuf.$(OBJEXT) send_stream.$(OBJEXT) \ + send_tcp.$(OBJEXT) send_udp.$(OBJEXT) send_udp_ibv.$(OBJEXT) libspead2_a_OBJECTS = $(am_libspead2_a_OBJECTS) @SPEAD2_USE_IBV_TRUE@am__EXEEXT_1 = mcdump$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) @@ -198,11 +198,16 @@ spead2_send_OBJECTS = $(am_spead2_send_OBJECTS) spead2_send_DEPENDENCIES = $(LDADD) am_spead2_unittest_OBJECTS = spead2_unittest-unittest_main.$(OBJEXT) \ + spead2_unittest-unittest_logging.$(OBJEXT) \ spead2_unittest-unittest_memcpy.$(OBJEXT) \ spead2_unittest-unittest_memory_allocator.$(OBJEXT) \ spead2_unittest-unittest_memory_pool.$(OBJEXT) \ + spead2_unittest-unittest_raw_packet.$(OBJEXT) \ spead2_unittest-unittest_recv_live_heap.$(OBJEXT) \ - spead2_unittest-unittest_send_heap.$(OBJEXT) + spead2_unittest-unittest_recv_custom_memcpy.$(OBJEXT) \ + spead2_unittest-unittest_semaphore.$(OBJEXT) \ + spead2_unittest-unittest_send_heap.$(OBJEXT) \ + spead2_unittest-unittest_send_streambuf.$(OBJEXT) spead2_unittest_OBJECTS = $(am_spead2_unittest_OBJECTS) spead2_unittest_DEPENDENCIES = $(LDADD) AM_V_P = $(am__v_P_@AM_V@) @@ -457,9 +462,6 @@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXDEPMODE = @CXXDEPMODE@ @@ -498,19 +500,19 @@ SPEAD2_USE_EVENTFD = @SPEAD2_USE_EVENTFD@ SPEAD2_USE_IBV = @SPEAD2_USE_IBV@ SPEAD2_USE_IBV_EXP = @SPEAD2_USE_IBV_EXP@ +SPEAD2_USE_IBV_MPRQ = @SPEAD2_USE_IBV_MPRQ@ SPEAD2_USE_MOVNTDQ = @SPEAD2_USE_MOVNTDQ@ -SPEAD2_USE_NETMAP = @SPEAD2_USE_NETMAP@ SPEAD2_USE_PCAP = @SPEAD2_USE_PCAP@ SPEAD2_USE_POSIX_SEMAPHORES = @SPEAD2_USE_POSIX_SEMAPHORES@ SPEAD2_USE_PTHREAD_SETAFFINITY_NP = @SPEAD2_USE_PTHREAD_SETAFFINITY_NP@ SPEAD2_USE_RECVMMSG = @SPEAD2_USE_RECVMMSG@ +SPEAD2_USE_SENDMMSG = @SPEAD2_USE_SENDMMSG@ STRIP = @STRIP@ VERSION = @VERSION@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ -ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ @@ -567,17 +569,24 @@ spead2_bench_LDADD = -lboost_program_options $(LDADD) spead2_unittest_SOURCES = \ unittest_main.cpp \ + unittest_logging.cpp \ unittest_memcpy.cpp \ unittest_memory_allocator.cpp \ unittest_memory_pool.cpp \ + unittest_raw_packet.cpp \ unittest_recv_live_heap.cpp \ - unittest_send_heap.cpp + unittest_recv_custom_memcpy.cpp \ + unittest_semaphore.cpp \ + unittest_send_heap.cpp \ + unittest_send_streambuf.cpp spead2_unittest_CPPFLAGS = -DBOOST_TEST_DYN_LINK $(AM_CPPFLAGS) spead2_unittest_LDADD = -lboost_unit_test_framework $(LDADD) libspead2_a_SOURCES = \ common_flavour.cpp \ common_ibv.cpp \ + common_ibv_loader.cpp \ + common_ibv_loader_utils.cpp \ common_inproc.cpp \ common_logging.cpp \ common_memcpy.cpp \ @@ -591,7 +600,6 @@ recv_inproc.cpp \ recv_live_heap.cpp \ recv_mem.cpp \ - recv_netmap.cpp \ recv_packet.cpp \ recv_reader.cpp \ recv_ring_stream.cpp \ @@ -600,6 +608,7 @@ recv_udp_base.cpp \ recv_udp.cpp \ recv_udp_ibv.cpp \ + recv_udp_ibv_mprq.cpp \ recv_udp_pcap.cpp \ send_heap.cpp \ send_inproc.cpp \ @@ -754,6 +763,8 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_flavour.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_ibv.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_ibv_loader.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_ibv_loader_utils.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_inproc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_logging.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/common_memcpy.Po@am__quote@ @@ -768,7 +779,6 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_inproc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_live_heap.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_mem.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_netmap.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_packet.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_reader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_ring_stream.Po@am__quote@ @@ -777,6 +787,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_udp.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_udp_base.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_udp_ibv.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_udp_ibv_mprq.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recv_udp_pcap.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/send_heap.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/send_inproc.Po@am__quote@ @@ -789,12 +800,17 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_bench.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_recv.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_send.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_logging.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_main.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_memcpy.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_memory_allocator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_memory_pool.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_raw_packet.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_recv_live_heap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_semaphore.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_send_heap.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spead2_unittest-unittest_send_streambuf.Po@am__quote@ .cpp.o: @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< @@ -824,6 +840,20 @@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_main.obj `if test -f 'unittest_main.cpp'; then $(CYGPATH_W) 'unittest_main.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_main.cpp'; fi` +spead2_unittest-unittest_logging.o: unittest_logging.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_logging.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_logging.Tpo -c -o spead2_unittest-unittest_logging.o `test -f 'unittest_logging.cpp' || echo '$(srcdir)/'`unittest_logging.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_logging.Tpo $(DEPDIR)/spead2_unittest-unittest_logging.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_logging.cpp' object='spead2_unittest-unittest_logging.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_logging.o `test -f 'unittest_logging.cpp' || echo '$(srcdir)/'`unittest_logging.cpp + +spead2_unittest-unittest_logging.obj: unittest_logging.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_logging.obj -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_logging.Tpo -c -o spead2_unittest-unittest_logging.obj `if test -f 'unittest_logging.cpp'; then $(CYGPATH_W) 'unittest_logging.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_logging.cpp'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_logging.Tpo $(DEPDIR)/spead2_unittest-unittest_logging.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_logging.cpp' object='spead2_unittest-unittest_logging.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_logging.obj `if test -f 'unittest_logging.cpp'; then $(CYGPATH_W) 'unittest_logging.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_logging.cpp'; fi` + spead2_unittest-unittest_memcpy.o: unittest_memcpy.cpp @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_memcpy.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_memcpy.Tpo -c -o spead2_unittest-unittest_memcpy.o `test -f 'unittest_memcpy.cpp' || echo '$(srcdir)/'`unittest_memcpy.cpp @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_memcpy.Tpo $(DEPDIR)/spead2_unittest-unittest_memcpy.Po @@ -866,6 +896,20 @@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_memory_pool.obj `if test -f 'unittest_memory_pool.cpp'; then $(CYGPATH_W) 'unittest_memory_pool.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_memory_pool.cpp'; fi` +spead2_unittest-unittest_raw_packet.o: unittest_raw_packet.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_raw_packet.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_raw_packet.Tpo -c -o spead2_unittest-unittest_raw_packet.o `test -f 'unittest_raw_packet.cpp' || echo '$(srcdir)/'`unittest_raw_packet.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_raw_packet.Tpo $(DEPDIR)/spead2_unittest-unittest_raw_packet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_raw_packet.cpp' object='spead2_unittest-unittest_raw_packet.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_raw_packet.o `test -f 'unittest_raw_packet.cpp' || echo '$(srcdir)/'`unittest_raw_packet.cpp + +spead2_unittest-unittest_raw_packet.obj: unittest_raw_packet.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_raw_packet.obj -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_raw_packet.Tpo -c -o spead2_unittest-unittest_raw_packet.obj `if test -f 'unittest_raw_packet.cpp'; then $(CYGPATH_W) 'unittest_raw_packet.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_raw_packet.cpp'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_raw_packet.Tpo $(DEPDIR)/spead2_unittest-unittest_raw_packet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_raw_packet.cpp' object='spead2_unittest-unittest_raw_packet.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_raw_packet.obj `if test -f 'unittest_raw_packet.cpp'; then $(CYGPATH_W) 'unittest_raw_packet.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_raw_packet.cpp'; fi` + spead2_unittest-unittest_recv_live_heap.o: unittest_recv_live_heap.cpp @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_recv_live_heap.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_recv_live_heap.Tpo -c -o spead2_unittest-unittest_recv_live_heap.o `test -f 'unittest_recv_live_heap.cpp' || echo '$(srcdir)/'`unittest_recv_live_heap.cpp @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_recv_live_heap.Tpo $(DEPDIR)/spead2_unittest-unittest_recv_live_heap.Po @@ -880,6 +924,34 @@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_recv_live_heap.obj `if test -f 'unittest_recv_live_heap.cpp'; then $(CYGPATH_W) 'unittest_recv_live_heap.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_recv_live_heap.cpp'; fi` +spead2_unittest-unittest_recv_custom_memcpy.o: unittest_recv_custom_memcpy.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_recv_custom_memcpy.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Tpo -c -o spead2_unittest-unittest_recv_custom_memcpy.o `test -f 'unittest_recv_custom_memcpy.cpp' || echo '$(srcdir)/'`unittest_recv_custom_memcpy.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Tpo $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_recv_custom_memcpy.cpp' object='spead2_unittest-unittest_recv_custom_memcpy.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_recv_custom_memcpy.o `test -f 'unittest_recv_custom_memcpy.cpp' || echo '$(srcdir)/'`unittest_recv_custom_memcpy.cpp + +spead2_unittest-unittest_recv_custom_memcpy.obj: unittest_recv_custom_memcpy.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_recv_custom_memcpy.obj -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Tpo -c -o spead2_unittest-unittest_recv_custom_memcpy.obj `if test -f 'unittest_recv_custom_memcpy.cpp'; then $(CYGPATH_W) 'unittest_recv_custom_memcpy.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_recv_custom_memcpy.cpp'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Tpo $(DEPDIR)/spead2_unittest-unittest_recv_custom_memcpy.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_recv_custom_memcpy.cpp' object='spead2_unittest-unittest_recv_custom_memcpy.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_recv_custom_memcpy.obj `if test -f 'unittest_recv_custom_memcpy.cpp'; then $(CYGPATH_W) 'unittest_recv_custom_memcpy.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_recv_custom_memcpy.cpp'; fi` + +spead2_unittest-unittest_semaphore.o: unittest_semaphore.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_semaphore.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_semaphore.Tpo -c -o spead2_unittest-unittest_semaphore.o `test -f 'unittest_semaphore.cpp' || echo '$(srcdir)/'`unittest_semaphore.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_semaphore.Tpo $(DEPDIR)/spead2_unittest-unittest_semaphore.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_semaphore.cpp' object='spead2_unittest-unittest_semaphore.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_semaphore.o `test -f 'unittest_semaphore.cpp' || echo '$(srcdir)/'`unittest_semaphore.cpp + +spead2_unittest-unittest_semaphore.obj: unittest_semaphore.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_semaphore.obj -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_semaphore.Tpo -c -o spead2_unittest-unittest_semaphore.obj `if test -f 'unittest_semaphore.cpp'; then $(CYGPATH_W) 'unittest_semaphore.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_semaphore.cpp'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_semaphore.Tpo $(DEPDIR)/spead2_unittest-unittest_semaphore.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_semaphore.cpp' object='spead2_unittest-unittest_semaphore.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_semaphore.obj `if test -f 'unittest_semaphore.cpp'; then $(CYGPATH_W) 'unittest_semaphore.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_semaphore.cpp'; fi` + spead2_unittest-unittest_send_heap.o: unittest_send_heap.cpp @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_send_heap.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_send_heap.Tpo -c -o spead2_unittest-unittest_send_heap.o `test -f 'unittest_send_heap.cpp' || echo '$(srcdir)/'`unittest_send_heap.cpp @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_send_heap.Tpo $(DEPDIR)/spead2_unittest-unittest_send_heap.Po @@ -894,6 +966,20 @@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_send_heap.obj `if test -f 'unittest_send_heap.cpp'; then $(CYGPATH_W) 'unittest_send_heap.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_send_heap.cpp'; fi` +spead2_unittest-unittest_send_streambuf.o: unittest_send_streambuf.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_send_streambuf.o -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Tpo -c -o spead2_unittest-unittest_send_streambuf.o `test -f 'unittest_send_streambuf.cpp' || echo '$(srcdir)/'`unittest_send_streambuf.cpp +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Tpo $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_send_streambuf.cpp' object='spead2_unittest-unittest_send_streambuf.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_send_streambuf.o `test -f 'unittest_send_streambuf.cpp' || echo '$(srcdir)/'`unittest_send_streambuf.cpp + +spead2_unittest-unittest_send_streambuf.obj: unittest_send_streambuf.cpp +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spead2_unittest-unittest_send_streambuf.obj -MD -MP -MF $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Tpo -c -o spead2_unittest-unittest_send_streambuf.obj `if test -f 'unittest_send_streambuf.cpp'; then $(CYGPATH_W) 'unittest_send_streambuf.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_send_streambuf.cpp'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Tpo $(DEPDIR)/spead2_unittest-unittest_send_streambuf.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='unittest_send_streambuf.cpp' object='spead2_unittest-unittest_send_streambuf.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spead2_unittest_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spead2_unittest-unittest_send_streambuf.obj `if test -f 'unittest_send_streambuf.cpp'; then $(CYGPATH_W) 'unittest_send_streambuf.cpp'; else $(CYGPATH_W) '$(srcdir)/unittest_send_streambuf.cpp'; fi` + ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique tags: tags-am diff -Nru spead2-1.10.0/src/mcdump.cpp spead2-2.1.0/src/mcdump.cpp --- spead2-1.10.0/src/mcdump.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/mcdump.cpp 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016, 2017 SKA South Africa +/* Copyright 2016-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -17,8 +17,8 @@ /** * @file * - * Utility program to dump raw multicast packets, using ibverbs. It works with - * any multicast UDP data, not just SPEAD. + * Utility program to dump raw packets, using ibverbs. It works with any UDP + * data, not just SPEAD. * * The design is based on three threads: * 1. A network thread that handles the ibverbs interface, removing packets @@ -27,7 +27,7 @@ * 2. A collector thread that receives batches of packets and copies them * into new page-aligned buffers. * 3. A disk thread that writes page-aligned buffers to disk (if requested, - * using O_DIRECT). + * using O_DIRECT). There can actually be more than one of these. * * If no filename is given, the latter two threads are disabled. */ @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -140,7 +141,7 @@ hidden.add_options() ("filename", make_opt_nodefault(opts.filename), "output filename") - ("endpoint", po::value>(&opts.endpoints)->composing(), "multicast-group:port") + ("endpoint", po::value>(&opts.endpoints)->composing(), "address:port") ; all.add(desc); all.add(hidden); @@ -197,6 +198,9 @@ std::uint32_t orig_len; }; +/* Data associated with a single packet. When using a multi-packet receive + * queue, wr and sg are not used. + */ struct chunk_entry { ibv_recv_wr wr; @@ -204,14 +208,16 @@ record_header record; }; +/* A contiguous chunk of memory holding packets, before collection. */ struct chunk { std::uint32_t n_records; std::size_t n_bytes; + bool full; ///< No more work requests referencing it std::unique_ptr entries; std::unique_ptr iov; spead2::memory_pool::pointer storage; - spead2::ibv_mr_t records_mr, storage_mr; + spead2::ibv_mr_t storage_mr; }; static std::atomic stop{false}; @@ -315,7 +321,7 @@ { free_ring.pop(); } - catch (spead2::ringbuffer_stopped) + catch (spead2::ringbuffer_stopped &) { break; } @@ -381,7 +387,7 @@ b.length = 0; free_ring.push(std::move(b)); } - catch (spead2::ringbuffer_stopped) + catch (spead2::ringbuffer_stopped &) { break; } @@ -394,30 +400,54 @@ } } -class capture +struct chunking_scheme +{ + std::size_t max_records; ///< Packets per chunk + std::size_t n_chunks; ///< Number of chunks + std::size_t chunk_size; ///< Bytes per chunk +}; + +typedef std::function chunking_scheme_generator; + +/* Joins multicast groups for its lifetime */ +class joiner { private: + boost::asio::io_service io_service; + boost::asio::ip::udp::socket join_socket; + +public: + joiner(const boost::asio::ip::address_v4 &interface_address, + const std::vector &endpoints); +}; + +joiner::joiner(const boost::asio::ip::address_v4 &interface_address, + const std::vector &endpoints) + : join_socket(io_service, endpoints[0].protocol()) +{ + join_socket.set_option(boost::asio::socket_base::reuse_address(true)); + for (const auto &endpoint : endpoints) + if (endpoint.address().is_multicast()) + { + join_socket.set_option(boost::asio::ip::multicast::join_group( + endpoint.address().to_v4(), interface_address)); + } +} + +class capture_base +{ +protected: typedef spead2::ringbuffer ringbuffer; - const options opts; - std::size_t max_records; std::unique_ptr w; - ringbuffer ring; - ringbuffer free_ring; + const options opts; + + boost::asio::ip::address_v4 interface_address; spead2::rdma_event_channel_t event_channel; spead2::rdma_cm_id_t cm_id; - spead2::ibv_qp_t qp; spead2::ibv_pd_t pd; - spead2::ibv_cq_t cq; - std::vector flows; - std::uint64_t errors = 0; - std::uint64_t packets = 0; - std::uint64_t bytes = 0; - time_point start_time; - time_point last_report; - std::uint64_t last_errors = 0; - std::uint64_t last_packets = 0; - std::uint64_t last_bytes = 0; + const chunking_scheme chunking; #if SPEAD2_USE_IBV_EXP bool timestamp_support = false; @@ -427,29 +457,51 @@ std::uint64_t timestamp_add = 0; // initial system timestamp in ns #endif + ringbuffer ring; + ringbuffer free_ring; + std::uint64_t errors = 0; + std::uint64_t packets = 0; + std::uint64_t bytes = 0; + time_point start_time; + time_point last_report; + std::uint64_t last_errors = 0; + std::uint64_t last_packets = 0; + std::uint64_t last_bytes = 0; + + void init_timestamp_support(); chunk make_chunk(spead2::memory_allocator &allocator); + void add_to_free(chunk &&c); void chunk_done(chunk &&c); - void report_rates(time_point now); void collect_thread(); - void network_thread(); + + virtual void network_thread() = 0; + virtual void post_chunk(chunk &c) = 0; + virtual void init_record(chunk &c, std::size_t idx) = 0; + + void chunk_ready(chunk &&c); + void report_rates(time_point now); + + capture_base(const options &opts, const chunking_scheme_generator &gen_chunking); public: - capture(const options &opts); - ~capture(); + virtual ~capture_base(); void run(); }; -chunk capture::make_chunk(spead2::memory_allocator &allocator) +chunk capture_base::make_chunk(spead2::memory_allocator &allocator) { + const std::size_t max_records = chunking.max_records; + const std::size_t chunk_size = chunking.chunk_size; chunk c; c.n_records = 0; c.n_bytes = 0; + c.full = false; c.entries.reset(new chunk_entry[max_records]); c.iov.reset(new iovec[2 * max_records]); - c.storage = allocator.allocate(opts.snaplen * max_records, nullptr); - c.storage_mr = spead2::ibv_mr_t(pd, c.storage.get(), opts.snaplen * max_records, IBV_ACCESS_LOCAL_WRITE); + c.storage = allocator.allocate(chunk_size, nullptr); + c.storage_mr = spead2::ibv_mr_t(pd, c.storage.get(), chunk_size, IBV_ACCESS_LOCAL_WRITE); std::uintptr_t ptr = (std::uintptr_t) c.storage.get(); for (std::uint32_t i = 0; i < max_records; i++) { @@ -468,15 +520,16 @@ return c; } -void capture::add_to_free(chunk &&c) +void capture_base::add_to_free(chunk &&c) { c.n_records = 0; c.n_bytes = 0; - qp.post_recv(&c.entries[0].wr); + c.full = false; + post_chunk(c); free_ring.push(std::move(c)); } -void capture::chunk_done(chunk &&c) +void capture_base::chunk_done(chunk &&c) { /* Only post a new receive if the chunk was full. If it was * not full, then this was the last chunk, and we're about to @@ -488,13 +541,21 @@ * chunk. So we push it back onto the ring without posting a * new receive, just to keep it live. */ - if (c.n_records == max_records) + if (c.full) add_to_free(std::move(c)); else free_ring.push(std::move(c)); } -void capture::collect_thread() +void capture_base::chunk_ready(chunk &&c) +{ + if (opts.filename != "-") + ring.push(std::move(c)); + else + chunk_done(std::move(c)); +} + +void capture_base::collect_thread() { try { @@ -515,7 +576,7 @@ chunk_done(std::move(c)); } - catch (spead2::ringbuffer_stopped) + catch (spead2::ringbuffer_stopped &) { free_ring.stop(); w->close(); @@ -530,7 +591,7 @@ } } -void capture::report_rates(time_point now) +void capture_base::report_rates(time_point now) { boost::format formatter("B: %|-14| P: %|-12| MB/s: %|-10.0f| kP/s: %|-10.0f|\n"); std::chrono::duration elapsed = now - last_report; @@ -544,17 +605,286 @@ last_report = now; } +static boost::asio::ip::address_v4 get_interface_address(const options &opts) +{ + boost::asio::ip::address_v4 interface_address; + try + { + interface_address = boost::asio::ip::address_v4::from_string(opts.interface); + } + catch (std::exception &) + { + throw std::runtime_error("Invalid interface address " + opts.interface); + } + return interface_address; +} + +static spead2::rdma_cm_id_t create_cm_id(const boost::asio::ip::address_v4 &interface_address, + const spead2::rdma_event_channel_t &event_channel) +{ + spead2::rdma_cm_id_t cm_id(event_channel, nullptr, RDMA_PS_UDP); + cm_id.bind_addr(interface_address); + return cm_id; +} + +capture_base::capture_base(const options &opts, const chunking_scheme_generator &gen_chunking) + : opts(opts), + interface_address(get_interface_address(opts)), + cm_id(create_cm_id(interface_address, event_channel)), + pd(cm_id), + chunking(gen_chunking(opts, cm_id)), + ring(chunking.n_chunks), + free_ring(chunking.n_chunks) +{ + init_timestamp_support(); +} + +capture_base::~capture_base() +{ + // This is needed (in the error case) to unblock the writer threads so that + // shutdown doesn't deadlock. + if (w) + w->close(); +} + +void capture_base::init_timestamp_support() +{ +#if SPEAD2_USE_IBV_EXP + timestamp_support = true; + // Get tick rate of the HW timestamp counter + { + ibv_exp_device_attr device_attr = {}; + device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK; + int ret = ibv_exp_query_device(cm_id->verbs, &device_attr); + /* There is some confusion about the units of hca_core_clock. Mellanox + * documentation indicates it is in MHz + * (http://www.mellanox.com/related-docs/prod_software/Mellanox_OFED_Linux_User_Manual_v4.1.pdf) + * but their implementation in OFED 4.1 returns kHz. We work in kHz but if + * if the value is suspiciously small (< 10 MHz) we assume the units were MHz. + */ + if (device_attr.hca_core_clock < 10000) + device_attr.hca_core_clock *= 1000; + if (ret == 0 && (device_attr.comp_mask & IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK)) + hca_ns_per_clock = 1e6 / device_attr.hca_core_clock; + else + { + spead2::log_warning("could not query hca_core_clock: timestamps disabled"); + timestamp_support = false; + } + } + // Get current value of HW timestamp counter, to subtract + if (timestamp_support) + { + ibv_exp_values values = {}; + values.comp_mask = IBV_EXP_VALUES_HW_CLOCK; + int ret = ibv_exp_query_values(cm_id->verbs, IBV_EXP_VALUES_HW_CLOCK, &values); + if (ret == 0 && (values.comp_mask & IBV_EXP_VALUES_HW_CLOCK)) + { + timestamp_subtract = values.hwclock; + timespec now; + clock_gettime(CLOCK_REALTIME, &now); + timestamp_add = now.tv_sec * std::uint64_t(1000000000) + now.tv_nsec; + } + else + { + spead2::log_warning("could not query current HW time: timestamps disabled"); + timestamp_support = false; + } + } +#endif +} + +static boost::asio::ip::udp::endpoint make_endpoint(const std::string &s) +{ + // Use rfind rather than find because IPv6 addresses contain :'s + auto pos = s.rfind(':'); + if (pos == std::string::npos) + { + throw std::runtime_error("Endpoint " + s + " is missing a port number"); + } + try + { + boost::asio::ip::address_v4 addr = boost::asio::ip::address_v4::from_string(s.substr(0, pos)); + std::uint16_t port = boost::lexical_cast(s.substr(pos + 1)); + return boost::asio::ip::udp::endpoint(addr, port); + } + catch (boost::bad_lexical_cast &) + { + throw std::runtime_error("Invalid port number " + s.substr(pos + 1)); + } +} + +void capture_base::run() +{ + using boost::asio::ip::udp; + + std::shared_ptr allocator = + std::make_shared(0, true); + bool has_file = opts.filename != "-"; + if (has_file) + { + int fd_flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef O_DIRECT + if (opts.direct) + fd_flags |= O_DIRECT; +#endif + int fd = open(opts.filename.c_str(), fd_flags, 0666); + if (fd < 0) + spead2::throw_errno("open failed"); + w.reset(new writer(opts, fd, *allocator)); + } + + /* Run in a thread that's bound to network_affinity, so that the pages are more likely + * to end up on the right NUMA node. It is system policy dependent, the + * Linux default is to allocate memory on the same node as the CPU that + * made the allocation. + * + * We don't want to bind the parent thread, because we haven't yet forked + * off collect_thread, and if it doesn't have a specific affinity we don't + * want it to inherit network_affinity. + */ + std::future alloc_future = std::async(std::launch::async, [this, allocator] { + if (opts.network_affinity >= 0) + spead2::thread_pool::set_affinity(opts.network_affinity); + for (std::size_t i = 0; i < chunking.n_chunks; i++) + add_to_free(make_chunk(*allocator)); + }); + alloc_future.get(); + + struct sigaction act = {}, old_act; + act.sa_handler = signal_handler; + act.sa_flags = SA_RESETHAND | SA_RESTART; + int ret = sigaction(SIGINT, &act, &old_act); + if (ret != 0) + spead2::throw_errno("sigaction failed"); + + std::future collect_future; + if (has_file) + collect_future = std::async(std::launch::async, [this] { collect_thread(); }); + + if (opts.network_affinity >= 0) + spead2::thread_pool::set_affinity(opts.network_affinity); + try + { + network_thread(); + ring.stop(); + + /* Briefly sleep so that we can unsubscribe from the switch before we shut + * down the QP. This makes it more likely that we can avoid incrementing + * the dropped packets counter on the NIC. + */ + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + if (has_file) + collect_future.get(); + // Restore SIGINT handler + sigaction(SIGINT, &old_act, &act); + time_point now = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = now - start_time; + std::cout << "\n\n" << packets << " packets captured (" << bytes << " bytes) in " + << elapsed.count() << "s\n" + << errors << " errors\n"; + } + catch (...) + { + ring.stop(); + sigaction(SIGINT, &old_act, &act); + throw; + } +} + +class capture : public capture_base +{ +private: + spead2::ibv_cq_t cq; + spead2::ibv_qp_t qp; + + virtual void network_thread() override; + virtual void post_chunk(chunk &c) override; + virtual void init_record(chunk &c, std::size_t idx) override; + + static chunking_scheme sizes(const options &opts, const spead2::rdma_cm_id_t &cm_id); + +public: + capture(const options &opts); +}; + +capture::capture(const options &opts) + : capture_base(opts, sizes) +{ + std::uint32_t n_slots = chunking.n_chunks * chunking.max_records; +#if SPEAD2_USE_IBV_EXP + ibv_exp_cq_init_attr cq_attr = {}; + if (timestamp_support) + cq_attr.flags |= IBV_EXP_CQ_TIMESTAMP; + cq_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS; + cq = spead2::ibv_cq_t(cm_id, n_slots, nullptr, &cq_attr); +#else + cq = spead2::ibv_cq_t(cm_id, n_slots, nullptr); +#endif + + ibv_qp_init_attr qp_attr = {}; + qp_attr.send_cq = cq.get(); + qp_attr.recv_cq = cq.get(); + qp_attr.qp_type = IBV_QPT_RAW_PACKET; + qp_attr.cap.max_send_wr = 1; + qp_attr.cap.max_recv_wr = n_slots; + qp_attr.cap.max_send_sge = 1; + qp_attr.cap.max_recv_sge = 1; + qp = spead2::ibv_qp_t(pd, &qp_attr); + qp.modify(IBV_QPS_INIT, cm_id->port_num); +} + +chunking_scheme capture::sizes(const options &opts, const spead2::rdma_cm_id_t &cm_id) +{ + constexpr std::size_t nominal_chunk_size = 2 * 1024 * 1024; // TODO: make tunable? + std::size_t max_records = nominal_chunk_size / opts.snaplen; + if (max_records == 0) + max_records = 1; + + std::size_t chunk_size = max_records * opts.snaplen; + std::size_t n_chunks = opts.net_buffer / chunk_size; + if (n_chunks == 0) + n_chunks = 1; + + ibv_device_attr attr = cm_id.query_device(); + unsigned int device_slots = std::min(attr.max_cqe, attr.max_qp_wr); + unsigned int device_chunks = device_slots / max_records; + if (attr.max_mr < device_chunks) + device_chunks = attr.max_mr; + + bool reduced = false; + if (device_slots < max_records) + { + max_records = device_slots; + reduced = true; + } + if (device_slots < n_chunks * max_records) + { + n_chunks = device_slots / max_records; + reduced = true; + } + if (reduced) + spead2::log_warning("Reducing buffer to %d to accommodate device limits", + n_chunks * max_records * opts.snaplen); + return {max_records, n_chunks, opts.snaplen * max_records}; +} + void capture::network_thread() { // Number of polls + packets between checking the time. Fetching the // time is expensive, so we don't want to do it too often. constexpr int GET_TIME_RATE = 10000; - if (opts.network_affinity >= 0) - spead2::thread_pool::set_affinity(opts.network_affinity); + std::vector endpoints; + for (const std::string &s : opts.endpoints) + endpoints.push_back(make_endpoint(s)); + auto flows = spead2::create_flows(qp, endpoints, cm_id->port_num); + qp.modify(IBV_QPS_RTR); + joiner join(interface_address, endpoints); + start_time = std::chrono::high_resolution_clock::now(); last_report = start_time; - bool has_file = opts.filename != "-"; + const std::size_t max_records = chunking.max_records; #if SPEAD2_USE_IBV_EXP std::unique_ptr wc(new ibv_exp_wc[max_records]); #else @@ -624,262 +954,296 @@ if (remaining_count != std::numeric_limits::max()) remaining_count -= n; } - if (has_file) - ring.push(std::move(c)); - else - chunk_done(std::move(c)); + c.full = c.n_records == max_records; + chunk_ready(std::move(c)); } - ring.stop(); } -static spead2::ibv_flow_t create_flow( - const spead2::ibv_qp_t &qp, const boost::asio::ip::udp::endpoint &endpoint, int port_num) +void capture::post_chunk(chunk &c) { - struct - { - ibv_flow_attr attr; - ibv_flow_spec_eth eth; - ibv_flow_spec_ipv4 ip; - ibv_flow_spec_tcp_udp udp; - } __attribute__((packed)) flow_rule; - memset(&flow_rule, 0, sizeof(flow_rule)); - - flow_rule.attr.type = IBV_FLOW_ATTR_NORMAL; - flow_rule.attr.priority = 0; - flow_rule.attr.size = sizeof(flow_rule); - flow_rule.attr.num_of_specs = 3; - flow_rule.attr.port = port_num; - - flow_rule.eth.type = IBV_FLOW_SPEC_ETH; - flow_rule.eth.size = sizeof(flow_rule.eth); - spead2::mac_address dst_mac = spead2::multicast_mac(endpoint.address()); - std::memcpy(&flow_rule.eth.val.dst_mac, &dst_mac, sizeof(dst_mac)); - // Set all 1's mask - std::memset(&flow_rule.eth.mask.dst_mac, 0xFF, sizeof(flow_rule.eth.mask.dst_mac)); - - flow_rule.ip.type = IBV_FLOW_SPEC_IPV4; - flow_rule.ip.size = sizeof(flow_rule.ip); - auto bytes = endpoint.address().to_v4().to_bytes(); // big-endian address - std::memcpy(&flow_rule.ip.val.dst_ip, &bytes, sizeof(bytes)); - std::memset(&flow_rule.ip.mask.dst_ip, 0xFF, sizeof(flow_rule.ip.mask.dst_ip)); - - flow_rule.udp.type = IBV_FLOW_SPEC_UDP; - flow_rule.udp.size = sizeof(flow_rule.udp); - flow_rule.udp.val.dst_port = htobe16(endpoint.port()); - flow_rule.udp.mask.dst_port = 0xFFFF; - - return spead2::ibv_flow_t(qp, &flow_rule.attr); -} - -static spead2::ibv_qp_t create_qp( - const spead2::ibv_pd_t &pd, const spead2::ibv_cq_t &cq, std::uint32_t n_slots) -{ - ibv_qp_init_attr attr; - memset(&attr, 0, sizeof(attr)); - attr.send_cq = cq.get(); - attr.recv_cq = cq.get(); - attr.qp_type = IBV_QPT_RAW_PACKET; - attr.cap.max_send_wr = 1; - attr.cap.max_recv_wr = n_slots; - attr.cap.max_send_sge = 1; - attr.cap.max_recv_sge = 1; - return spead2::ibv_qp_t(pd, &attr); + qp.post_recv(&c.entries[0].wr); } -// Returns number of records per chunk and number of chunks -static std::pair sizes(const options &opts) +void capture::init_record(chunk &c, std::size_t idx) { - constexpr std::size_t nominal_chunk_size = 2 * 1024 * 1024; // TODO: make tunable? - std::size_t max_records = nominal_chunk_size / opts.snaplen; - if (max_records == 0) - max_records = 1; - std::size_t chunk_size = max_records * opts.snaplen; - std::size_t n_chunks = opts.net_buffer / chunk_size; - if (n_chunks == 0) - n_chunks++; - return {max_records, n_chunks}; + std::uintptr_t ptr = (std::uintptr_t) c.storage.get(); + ptr += idx * opts.snaplen; + + c.entries[idx].wr.wr_id = idx; + c.entries[idx].wr.next = (idx + 1 < chunking.max_records) ? &c.entries[idx + 1].wr : nullptr; + c.entries[idx].wr.num_sge = 1; + c.entries[idx].wr.sg_list = &c.entries[idx].sg; + c.entries[idx].sg.addr = ptr; + c.entries[idx].sg.length = opts.snaplen; + c.entries[idx].sg.lkey = c.storage_mr->lkey; + c.iov[2 * idx].iov_base = &c.entries[idx].record; + c.iov[2 * idx].iov_len = sizeof(record_header); + c.iov[2 * idx + 1].iov_base = (void *) ptr; } -capture::capture(const options &opts) - : opts(opts), max_records(sizes(opts).first), - ring(sizes(opts).second), - free_ring(sizes(opts).second) +#if SPEAD2_USE_IBV_MPRQ + +class capture_mprq : public capture_base { -} +private: + spead2::ibv_exp_res_domain_t res_domain; + spead2::ibv_cq_t cq; + spead2::ibv_exp_cq_family_v1_t cq_intf; + spead2::ibv_exp_wq_t wq; + spead2::ibv_exp_wq_family_t wq_intf; + spead2::ibv_exp_rwq_ind_table_t rwq_ind_table; + spead2::ibv_qp_t qp; + + virtual void network_thread() override; + virtual void post_chunk(chunk &c) override; + virtual void init_record(chunk &c, std::size_t idx) override; + + static chunking_scheme sizes(const options &opts, const spead2::rdma_cm_id_t &cm_id); + +public: + capture_mprq(const options &opts); +}; -capture::~capture() +static int log2i(std::size_t value) { - // This is needed (in the error case) to unblock the writer threads so that - // shutdown doesn't deadlock. - if (w) - w->close(); + int x = 0; + while ((value >> x) != 1) + x++; + assert(value == std::size_t(1) << x); + return x; } -static boost::asio::ip::udp::endpoint make_endpoint(const std::string &s) +capture_mprq::capture_mprq(const options &opts) + : capture_base(opts, sizes) { - // Use rfind rather than find because IPv6 addresses contain :'s - auto pos = s.rfind(':'); - if (pos == std::string::npos) - { - throw std::runtime_error("Endpoint " + s + " is missing a port number"); - } - try - { - boost::asio::ip::address_v4 addr = boost::asio::ip::address_v4::from_string(s.substr(0, pos)); - if (!addr.is_multicast()) - throw std::runtime_error("Address " + s.substr(0, pos) + " is not a multicast address"); - std::uint16_t port = boost::lexical_cast(s.substr(pos + 1)); - return boost::asio::ip::udp::endpoint(addr, port); - } - catch (boost::bad_lexical_cast) + const std::size_t max_cqe = chunking.max_records * chunking.n_chunks; + + ibv_exp_res_domain_init_attr res_domain_attr = {}; + res_domain_attr.comp_mask = IBV_EXP_RES_DOMAIN_THREAD_MODEL | IBV_EXP_RES_DOMAIN_MSG_MODEL; + res_domain_attr.thread_model = IBV_EXP_THREAD_UNSAFE; + res_domain_attr.msg_model = IBV_EXP_MSG_HIGH_BW; + res_domain = spead2::ibv_exp_res_domain_t(cm_id, &res_domain_attr); + + ibv_exp_cq_init_attr cq_attr = {}; + if (timestamp_support) + cq_attr.flags |= IBV_EXP_CQ_TIMESTAMP; + cq_attr.res_domain = res_domain.get(); + cq_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS | IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN; + cq = spead2::ibv_cq_t(cm_id, max_cqe, nullptr, &cq_attr); + cq_intf = spead2::ibv_exp_cq_family_v1_t(cm_id, cq); + + ibv_exp_wq_init_attr wq_attr = {}; + std::size_t stride = chunking.chunk_size / chunking.max_records; + wq_attr.mp_rq.single_stride_log_num_of_bytes = log2i(stride); + wq_attr.mp_rq.single_wqe_log_num_of_strides = log2i(chunking.max_records); + wq_attr.mp_rq.use_shift = IBV_EXP_MP_RQ_NO_SHIFT; + wq_attr.wq_type = IBV_EXP_WQT_RQ; + wq_attr.max_recv_wr = chunking.n_chunks; + wq_attr.max_recv_sge = 1; + wq_attr.pd = pd.get(); + wq_attr.cq = cq.get(); + wq_attr.res_domain = res_domain.get(); + wq_attr.comp_mask = IBV_EXP_CREATE_WQ_MP_RQ | IBV_EXP_CREATE_WQ_RES_DOMAIN; + wq = spead2::ibv_exp_wq_t(cm_id, &wq_attr); + wq_intf = spead2::ibv_exp_wq_family_t(cm_id, wq); + + rwq_ind_table = spead2::create_rwq_ind_table(cm_id, pd, wq); + + /* ConnectX-5 only seems to work with Toeplitz hashing. This code seems to + * work, but I don't really know what I'm doing so it might be horrible. + */ + uint8_t toeplitz_key[40] = {}; + ibv_exp_rx_hash_conf hash_conf = {}; + hash_conf.rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ; + hash_conf.rx_hash_key_len = sizeof(toeplitz_key); + hash_conf.rx_hash_key = toeplitz_key; + hash_conf.rx_hash_fields_mask = 0; + hash_conf.rwq_ind_tbl = rwq_ind_table.get(); + + ibv_exp_qp_init_attr qp_attr = {}; + qp_attr.qp_type = IBV_QPT_RAW_PACKET; + qp_attr.pd = pd.get(); + qp_attr.res_domain = res_domain.get(); + qp_attr.rx_hash_conf = &hash_conf; + qp_attr.port_num = cm_id->port_num; + qp_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_RX_HASH + | IBV_EXP_QP_INIT_ATTR_PORT | IBV_EXP_QP_INIT_ATTR_RES_DOMAIN; + qp = spead2::ibv_qp_t(cm_id, &qp_attr); + + wq.modify(IBV_EXP_WQS_RDY); +} + +static int clamp(int x, int low, int high) +{ + return std::min(std::max(x, low), high); +} + +chunking_scheme capture_mprq::sizes(const options &opts, const spead2::rdma_cm_id_t &cm_id) +{ + ibv_exp_device_attr attr = cm_id.exp_query_device(); + if (!(attr.comp_mask & IBV_EXP_DEVICE_ATTR_MP_RQ) + || !(attr.mp_rq_caps.supported_qps & IBV_EXP_MP_RQ_SUP_TYPE_WQ_RQ)) + throw std::system_error(std::make_error_code(std::errc::not_supported), + "device does not support multi-packet receive queues"); + + /* TODO: adapt these to the requested buffer size e.g. if a very large + * buffer is requested, might need to increase the stride size to avoid + * running out of CQEs. + */ + std::size_t log_stride_bytes = + clamp(6, + attr.mp_rq_caps.min_single_stride_log_num_of_bytes, + attr.mp_rq_caps.max_single_stride_log_num_of_bytes); // 64 bytes + std::size_t log_strides_per_chunk = + clamp(21 - log_stride_bytes, + attr.mp_rq_caps.min_single_wqe_log_num_of_strides, + attr.mp_rq_caps.max_single_wqe_log_num_of_strides); // 2MB chunks + std::size_t max_records = 1 << log_strides_per_chunk; + std::size_t chunk_size = max_records << log_stride_bytes; + std::size_t n_chunks = opts.net_buffer / chunk_size; + if (n_chunks == 0) + n_chunks = 1; + + unsigned int device_chunks = std::min(attr.max_qp_wr, attr.max_mr); + + bool reduced = false; + if (device_chunks < n_chunks) { - throw std::runtime_error("Invalid port number " + s.substr(pos + 1)); + n_chunks = device_chunks; + reduced = true; } + + if (reduced) + spead2::log_warning("Reducing buffer to %d to accommodate device limits", + n_chunks * chunk_size); + return {max_records, n_chunks, chunk_size}; } -void capture::run() +void capture_mprq::network_thread() { - using boost::asio::ip::udp; - - std::shared_ptr allocator = - std::make_shared(0, true); - bool has_file = opts.filename != "-"; - if (has_file) - { - int fd_flags = O_WRONLY | O_CREAT | O_TRUNC; -#ifdef O_DIRECT - if (opts.direct) - fd_flags |= O_DIRECT; -#endif - int fd = open(opts.filename.c_str(), fd_flags, 0666); - if (fd < 0) - spead2::throw_errno("open failed"); - w.reset(new writer(opts, fd, *allocator)); - } + // Number of polls + packets between checking the time. Fetching the + // time is expensive, so we don't want to do it too often. + constexpr int GET_TIME_RATE = 10000; - boost::asio::io_service io_service; - std::vector endpoints; + std::vector endpoints; for (const std::string &s : opts.endpoints) endpoints.push_back(make_endpoint(s)); - boost::asio::ip::address_v4 interface_address; - try - { - interface_address = boost::asio::ip::address_v4::from_string(opts.interface); - } - catch (std::exception) - { - throw std::runtime_error("Invalid interface address " + opts.interface); - } + auto flows = spead2::create_flows(qp, endpoints, cm_id->port_num); + joiner join(interface_address, endpoints); - std::size_t n_chunks = sizes(opts).second; - if (std::numeric_limits::max() / max_records <= n_chunks) - throw std::runtime_error("Too many buffered packets"); - std::uint32_t n_slots = n_chunks * max_records; - cm_id = spead2::rdma_cm_id_t(event_channel, nullptr, RDMA_PS_UDP); - cm_id.bind_addr(interface_address); -#if SPEAD2_USE_IBV_EXP - timestamp_support = true; - // Get tick rate of the HW timestamp counter - { - ibv_exp_device_attr device_attr = {}; - device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK; - int ret = ibv_exp_query_device(cm_id->verbs, &device_attr); - /* There is some confusion about the units of hca_core_clock. Mellanox - * documentation indicates it is in MHz - * (http://www.mellanox.com/related-docs/prod_software/Mellanox_OFED_Linux_User_Manual_v4.1.pdf) - * but their implementation in OFED 4.1 returns kHz. We work in kHz but if - * if the value is suspiciously small (< 10 MHz) we assume the units were MHz. - */ - if (device_attr.hca_core_clock < 10000) - device_attr.hca_core_clock *= 1000; - if (ret == 0 && (device_attr.comp_mask & IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK)) - hca_ns_per_clock = 1e6 / device_attr.hca_core_clock; - else - { - spead2::log_warning("could not query hca_core_clock: timestamps disabled"); - timestamp_support = false; - } - } - // Get current value of HW timestamp counter, to subtract - if (timestamp_support) + start_time = std::chrono::high_resolution_clock::now(); + last_report = start_time; + const std::size_t max_records = chunking.max_records; + int until_get_time = GET_TIME_RATE; + std::uint64_t remaining_count = opts.count; + while (!stop.load() && remaining_count > 0) { - ibv_exp_values values = {}; - values.comp_mask = IBV_EXP_VALUES_HW_CLOCK; - int ret = ibv_exp_query_values(cm_id->verbs, IBV_EXP_VALUES_HW_CLOCK, &values); - if (ret == 0 && (values.comp_mask & IBV_EXP_VALUES_HW_CLOCK)) - { - timestamp_subtract = values.hwclock; - timespec now; - clock_gettime(CLOCK_REALTIME, &now); - timestamp_add = now.tv_sec * std::uint64_t(1000000000) + now.tv_nsec; - } - else + chunk c = free_ring.pop(); + while (!stop.load() && remaining_count > 0 && !c.full) { - spead2::log_warning("could not query current HW time: timestamps disabled"); - timestamp_support = false; + std::uint32_t offset; + std::uint32_t flags; + std::uint64_t timestamp; + std::int32_t len = cq_intf->poll_length_flags_mp_rq_ts( + cq.get(), &offset, &flags, ×tamp); + if (len < 0) + { + // Error condition. + ibv_wc wc; + ibv_poll_cq(cq.get(), 1, &wc); + spead2::log_warning("failed WR %1%: %2% (vendor_err: %3%)", + wc.wr_id, wc.status, wc.vendor_err); + errors++; + } + else if (len > 0) + { + packets++; + if (remaining_count != std::numeric_limits::max()) + remaining_count--; + std::size_t idx = c.n_records; + record_header &record = c.entries[idx].record; + record.incl_len = (len <= opts.snaplen) ? len : opts.snaplen; + record.orig_len = len; + if (timestamp_support && (flags & IBV_EXP_CQ_RX_WITH_TIMESTAMP)) + { + std::uint64_t ticks = timestamp - timestamp_subtract; + std::uint64_t ns = std::uint64_t(ticks * hca_ns_per_clock); + ns += timestamp_add; + record.ts_sec = ns / 1000000000; + record.ts_nsec = ns % 1000000000; + } + else + { + record.ts_sec = 0; + record.ts_nsec = 0; + } + c.iov[2 * idx + 1].iov_base = &c.storage[offset]; + c.iov[2 * idx + 1].iov_len = record.incl_len; + c.n_records++; + c.n_bytes += len + sizeof(record_header); + bytes += len; + } + until_get_time--; + if (until_get_time <= 0) + { + // TODO: unify this code with non-mprq + until_get_time = GET_TIME_RATE; + if (!opts.quiet) + { + time_point now = std::chrono::high_resolution_clock::now(); + if (now - last_report >= std::chrono::seconds(1)) + report_rates(now); + } + } + if (flags & IBV_EXP_CQ_RX_MULTI_PACKET_LAST_V1) + c.full = true; } + chunk_ready(std::move(c)); } +} - ibv_exp_cq_init_attr cq_attr = {}; - cq_attr.flags = IBV_EXP_CQ_TIMESTAMP; - cq_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS; - cq = spead2::ibv_cq_t(cm_id, n_slots, nullptr, &cq_attr); -#else - cq = spead2::ibv_cq_t(cm_id, n_slots, nullptr); -#endif - pd = spead2::ibv_pd_t(cm_id); - qp = create_qp(pd, cq, n_slots); - qp.modify(IBV_QPS_INIT, cm_id->port_num); - for (const udp::endpoint &endpoint : endpoints) - flows.push_back(create_flow(qp, endpoint, cm_id->port_num)); - - for (std::size_t i = 0; i < n_chunks; i++) - add_to_free(make_chunk(*allocator)); - qp.modify(IBV_QPS_RTR); - - struct sigaction act = {}, old_act; - act.sa_handler = signal_handler; - act.sa_flags = SA_RESETHAND | SA_RESTART; - int ret = sigaction(SIGINT, &act, &old_act); - if (ret != 0) - spead2::throw_errno("sigaction failed"); - - std::future collect_future; - if (has_file) - collect_future = std::async(std::launch::async, [this] { collect_thread(); }); +void capture_mprq::post_chunk(chunk &c) +{ + ibv_sge sge; + memset(&sge, 0, sizeof(sge)); + sge.addr = (uintptr_t) c.storage.get(); + sge.length = chunking.chunk_size; + sge.lkey = c.storage_mr->lkey; + int status = wq_intf->recv_burst(wq.get(), &sge, 1); + if (status != 0) + spead2::throw_errno("recv_burst failed", status); +} - udp::socket join_socket(io_service, endpoints[0].protocol()); - join_socket.set_option(boost::asio::socket_base::reuse_address(true)); - for (const udp::endpoint &endpoint : endpoints) - join_socket.set_option(boost::asio::ip::multicast::join_group( - endpoint.address().to_v4(), interface_address)); - - network_thread(); - /* Close socket then briefly sleep so that we can unsubscribe from the - * switch before we shut down the QP. This makes it more likely that we - * can avoid incrementing the dropped packets counter on the NIC. - */ - join_socket.close(); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - if (has_file) - collect_future.get(); - // Restore SIGINT handler - sigaction(SIGINT, &old_act, &act); - time_point now = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed = now - start_time; - std::cout << "\n\n" << packets << " packets captured (" << bytes << " bytes) in " - << elapsed.count() << "s\n" - << errors << " errors\n"; +void capture_mprq::init_record(chunk &c, std::size_t idx) +{ + c.iov[2 * idx].iov_base = &c.entries[idx].record; + c.iov[2 * idx].iov_len = sizeof(record_header); } +#endif // SPEAD2_USE_IBV_MPRQ + int main(int argc, const char **argv) { try { spead2::set_log_function(log_function); options opts = parse_args(argc, argv); - capture cap(opts); - cap.run(); + std::unique_ptr cap; + +#if SPEAD2_USE_IBV_MPRQ + try + { + cap.reset(new capture_mprq(opts)); + } + catch (std::system_error &e) + { + if (e.code() != std::errc::not_supported) + throw; + } +#endif + if (!cap) + cap.reset(new capture(opts)); + cap->run(); } catch (std::runtime_error &e) { diff -Nru spead2-1.10.0/src/py_common.cpp spead2-2.1.0/src/py_common.cpp --- spead2-1.10.0/src/py_common.cpp 2018-09-05 10:30:08.000000000 +0000 +++ spead2-2.1.0/src/py_common.cpp 2019-10-28 19:03:25.000000000 +0000 @@ -33,6 +33,9 @@ #include #include #include +#if SPEAD2_USE_IBV +# include +#endif namespace py = pybind11; @@ -89,6 +92,18 @@ template class socket_wrapper; template class socket_wrapper; +boost::asio::ip::address make_address_no_release( + boost::asio::io_service &io_service, const std::string &hostname, + boost::asio::ip::resolver_query_base::flags flags) +{ + if (hostname == "") + return boost::asio::ip::address(); + using boost::asio::ip::udp; + udp::resolver resolver(io_service); + udp::resolver::query query(hostname, "", flags); + return resolver.resolve(query)->endpoint().address(); +} + void deprecation_warning(const char *msg) { if (PyErr_WarnEx(PyExc_DeprecationWarning, msg, 1) == -1) @@ -158,7 +173,7 @@ log(msg.first, msg.second); } } - catch (ringbuffer_empty) + catch (ringbuffer_empty &) { } if (overflowed.exchange(false)) @@ -166,7 +181,7 @@ "Log ringbuffer was full - some log messages were dropped"); } } - catch (ringbuffer_stopped) + catch (ringbuffer_stopped &) { // Could possibly report the overflowed flag here again - but this may be // deep into interpreter shutdown and it might not be safe to log. @@ -204,7 +219,7 @@ { ring.try_emplace(level, msg); } - catch (ringbuffer_full) + catch (ringbuffer_full &) { overflowed = true; } @@ -225,7 +240,6 @@ void register_module(py::module m) { - using namespace spead2; using namespace pybind11::literals; py::register_exception(m, "Stopped"); @@ -340,6 +354,18 @@ .def_readwrite("format", &descriptor::format) .def_property("numpy_header", bytes_getter(&descriptor::numpy_header), bytes_setter(&descriptor::numpy_header)) ; +#if SPEAD2_USE_IBV + py::class_(m, "IbvContext") + .def(py::init([](const std::string &interface_address) + { + py::gil_scoped_release release; + boost::asio::io_service io_service; + return ibv_context_t(make_address_no_release( + io_service, interface_address, boost::asio::ip::udp::resolver::query::passive)); + }), "interface"_a) + .def("reset", [](ibv_context_t &self) { self.reset(); }) + ; +#endif } void register_logging() diff -Nru spead2-1.10.0/src/py_recv.cpp spead2-2.1.0/src/py_recv.cpp --- spead2-1.10.0/src/py_recv.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/py_recv.cpp 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -109,15 +109,8 @@ boost::asio::ip::address make_address(const std::string &hostname) { - if (hostname.empty()) - return boost::asio::ip::address_v4::any(); - else - { - using boost::asio::ip::udp; - udp::resolver resolver(get_strand().get_io_service()); - udp::resolver::query query(hostname, "", udp::resolver::query::passive); - return resolver.resolve(query)->endpoint().address(); - } + return make_address_no_release(get_io_service(), hostname, + boost::asio::ip::udp::resolver::query::passive); } template @@ -215,7 +208,7 @@ else { deprecation_warning("passing unbound socket plus port is deprecated"); - auto asio_socket = socket->copy(get_strand().get_io_service()); + auto asio_socket = socket->copy(get_io_service()); py::gil_scoped_release gil; auto endpoint = make_endpoint(bind_hostname, port); emplace_reader(std::move(asio_socket), endpoint, max_size, buffer_size); @@ -226,32 +219,32 @@ const socket_wrapper &socket, std::size_t max_size = udp_reader::default_max_size) { - auto asio_socket = socket.copy(get_strand().get_io_service()); + auto asio_socket = socket.copy(get_io_service()); py::gil_scoped_release gil; emplace_reader(std::move(asio_socket), max_size); } - void add_udp_reader_multicast_v4( - const std::string &multicast_group, + void add_udp_reader_bind_v4( + const std::string &address, std::uint16_t port, std::size_t max_size, std::size_t buffer_size, const std::string &interface_address) { py::gil_scoped_release gil; - auto endpoint = make_endpoint(multicast_group, port); + auto endpoint = make_endpoint(address, port); emplace_reader(endpoint, max_size, buffer_size, make_address(interface_address)); } - void add_udp_reader_multicast_v6( - const std::string &multicast_group, + void add_udp_reader_bind_v6( + const std::string &address, std::uint16_t port, std::size_t max_size, std::size_t buffer_size, unsigned int interface_index) { py::gil_scoped_release gil; - auto endpoint = make_endpoint(multicast_group, port); + auto endpoint = make_endpoint(address, port); emplace_reader(endpoint, max_size, buffer_size, interface_index); } @@ -270,14 +263,14 @@ const socket_wrapper &acceptor, std::size_t max_size) { - auto asio_socket = acceptor.copy(get_strand().get_io_service()); + auto asio_socket = acceptor.copy(get_io_service()); py::gil_scoped_release gil; emplace_reader(std::move(asio_socket), max_size); } #if SPEAD2_USE_IBV void add_udp_ibv_reader_single( - const std::string &multicast_group, + const std::string &address, std::uint16_t port, const std::string &interface_address, std::size_t max_size, @@ -286,7 +279,7 @@ int max_poll) { py::gil_scoped_release gil; - auto endpoint = make_endpoint(multicast_group, port); + auto endpoint = make_endpoint(address, port); emplace_reader(endpoint, make_address(interface_address), max_size, buffer_size, comp_vector, max_poll); } @@ -304,9 +297,9 @@ for (size_t i = 0; i < len(endpoints); i++) { py::sequence endpoint = endpoints[i].cast(); - std::string multicast_group = endpoint[0].cast(); + std::string address = endpoint[0].cast(); std::uint16_t port = endpoint[1].cast(); - endpoints2.push_back(make_endpoint(multicast_group, port)); + endpoints2.push_back(make_endpoint(address, port)); } py::gil_scoped_release gil; emplace_reader(endpoints2, make_address(interface_address), @@ -345,10 +338,8 @@ py::module register_module(py::module &parent) { using namespace pybind11::literals; - using namespace spead2::recv; - // Create the module, and set it as the current boost::python scope so that - // classes we define are added to this module rather than the root. + // Create the module py::module m = parent.def_submodule("recv"); py::class_(m, "HeapBase") @@ -422,6 +413,13 @@ [](ring_stream_wrapper &self, bool stop) { self.set_stop_on_stop_item(stop); }) + .def_property("allow_unsized_heaps", + [](const ring_stream_wrapper &self) { + return self.get_allow_unsized_heaps(); + }, + [](ring_stream_wrapper &self, bool allow) { + self.set_allow_unsized_heaps(allow); + }) .def("add_buffer_reader", SPEAD2_PTMF(ring_stream_wrapper, add_buffer_reader), "buffer"_a) .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader), "port"_a, @@ -432,13 +430,13 @@ .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader_socket), "socket"_a, "max_size"_a = udp_reader::default_max_size) - .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader_multicast_v4), + .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader_bind_v4), "multicast_group"_a, "port"_a, "max_size"_a = udp_reader::default_max_size, "buffer_size"_a = udp_reader::default_buffer_size, "interface_address"_a = "0.0.0.0") - .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader_multicast_v6), + .def("add_udp_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_reader_bind_v6), "multicast_group"_a, "port"_a, "max_size"_a = udp_reader::default_max_size, @@ -473,10 +471,12 @@ .def("add_udp_pcap_file_reader", SPEAD2_PTMF(ring_stream_wrapper, add_udp_pcap_file_reader), "filename"_a) #endif - .def("add_inproc_reader", SPEAD2_PTMF(ring_stream_wrapper, add_inproc_reader)) + .def("add_inproc_reader", SPEAD2_PTMF(ring_stream_wrapper, add_inproc_reader), + "queue"_a) .def("stop", SPEAD2_PTMF(ring_stream_wrapper, stop)) .def_property_readonly("fd", SPEAD2_PTMF(ring_stream_wrapper, get_fd)) - .def_property_readonly("stats", SPEAD2_PTMF(ring_stream_wrapper, get_stats)) + // SPEAD2_PTMF doesn't work for get_stats because it's defined in stream_base, which is a protected ancestor + .def_property_readonly("stats", [](const ring_stream_wrapper &stream) { return stream.get_stats(); }) .def_property_readonly("ringbuffer", SPEAD2_PTMF(ring_stream_wrapper, get_ringbuffer)) #if SPEAD2_USE_IBV .def_readonly_static("DEFAULT_UDP_IBV_MAX_SIZE", &udp_ibv_reader::default_max_size) diff -Nru spead2-1.10.0/src/py_send.cpp spead2-2.1.0/src/py_send.cpp --- spead2-1.10.0/src/py_send.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/py_send.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -87,7 +87,7 @@ boost::asio::buffers_end(pkt.buffers))); } -static py::object make_io_error(boost::system::error_code ec) +static py::object make_io_error(const boost::system::error_code &ec) { if (ec) { @@ -144,18 +144,29 @@ } }; +struct callback_item +{ + py::handle callback; + py::handle h; // heap: kept here because it can only be freed with the GIL + boost::system::error_code ec; + item_pointer_t bytes_transferred; +}; + +static void free_callback_items(const std::vector &callbacks) +{ + for (const callback_item &item : callbacks) + { + if (item.h) + item.h.dec_ref(); + if (item.callback) + item.callback.dec_ref(); + } +} + template class asyncio_stream_wrapper : public Base { private: - struct callback_item - { - py::handle callback; - py::handle h; // heap: kept here because it can only be freed with the GIL - boost::system::error_code ec; - item_pointer_t bytes_transferred; - }; - semaphore_gil sem; std::vector callbacks; std::mutex callbacks_mutex; @@ -216,19 +227,24 @@ callback(make_io_error(item.ec), item.bytes_transferred); } } - catch (std::exception &e) + catch (py::error_already_set &e) { - /* Clean up the remaining handles. Note that we only get here if - * things have gone very badly wrong, such as an out-of-memory - * error. + log_warning("send callback raised Python exception; expect deadlocks!"); + free_callback_items(current_callbacks); + throw; + } + catch (std::bad_alloc &e) + { + /* If we're out of memory we might not be able to construct a log + * message. Just rely on Python to report an error. */ - for (const callback_item &item : current_callbacks) - { - if (item.h) - item.h.dec_ref(); - if (item.callback) - item.callback.dec_ref(); - } + free_callback_items(current_callbacks); + throw; + } + catch (std::exception &e) + { + log_warning("unexpected error in process_callbacks: %1%", e.what()); + free_callback_items(current_callbacks); throw; } } @@ -246,14 +262,9 @@ static boost::asio::ip::address make_address( boost::asio::io_service &io_service, const std::string &hostname) { - if (hostname == "") - return boost::asio::ip::address(); py::gil_scoped_release gil; - - using boost::asio::ip::udp; - udp::resolver resolver(io_service); - udp::resolver::query query(hostname, "", boost::asio::ip::resolver_query_base::flags(0)); - return resolver.resolve(query)->endpoint().address(); + return make_address_no_release(io_service, hostname, + boost::asio::ip::resolver_query_base::flags(0)); } template @@ -405,7 +416,7 @@ using namespace pybind11::literals; return py::class_(m, name) - .def(py::init, std::string, std::uint16_t, const stream_config &, std::size_t, std::string>(), + .def(py::init, std::string, std::uint16_t, const stream_config &, std::size_t, const socket_wrapper &>(), "thread_pool"_a, "hostname"_a, "port"_a, "config"_a = stream_config(), "buffer_size"_a = T::default_buffer_size, @@ -539,7 +550,7 @@ static std::unique_ptr construct(Args... args) { std::shared_ptr state = std::make_shared(); - auto connect_handler = [state](boost::system::error_code ec) + auto connect_handler = [state](const boost::system::error_code &ec) { state->ec = ec; state->sem.put(); @@ -649,17 +660,19 @@ py::module register_module(py::module &parent) { using namespace pybind11::literals; - using namespace spead2::send; py::module m = parent.def_submodule("send"); py::class_(m, "Heap") .def(py::init(), "flavour"_a = flavour()) - .def_property_readonly("flavour", &heap_wrapper::get_flavour) + .def_property_readonly("flavour", SPEAD2_PTMF(heap_wrapper, get_flavour)) .def("add_item", SPEAD2_PTMF(heap_wrapper, add_item), "item"_a) .def("add_descriptor", SPEAD2_PTMF(heap_wrapper, add_descriptor), "descriptor"_a) .def("add_start", SPEAD2_PTMF(heap_wrapper, add_start)) - .def("add_end", SPEAD2_PTMF(heap_wrapper, add_end)); + .def("add_end", SPEAD2_PTMF(heap_wrapper, add_end)) + .def_property("repeat_pointers", + SPEAD2_PTMF(heap_wrapper, get_repeat_pointers), + SPEAD2_PTMF(heap_wrapper, set_repeat_pointers)); // keep_alive is safe to use here in spite of pybind/pybind11#856, because // the destructor of packet_generator doesn't reference the heap. diff -Nru spead2-1.10.0/src/recv_heap.cpp spead2-2.1.0/src/recv_heap.cpp --- spead2-1.10.0/src/recv_heap.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/recv_heap.cpp 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -49,7 +49,7 @@ return betoh(out); } -void heap_base::transfer_immediates(heap_base &&other) +void heap_base::transfer_immediates(heap_base &&other) noexcept { if (!immediate_payload) { @@ -61,7 +61,7 @@ } } -heap_base::heap_base(heap_base &&other) +heap_base::heap_base(heap_base &&other) noexcept : cnt(std::move(other.cnt)), flavour_(std::move(other.flavour_)), items(std::move(other.items)), @@ -71,7 +71,7 @@ transfer_immediates(std::move(other)); } -heap_base &heap_base::operator=(heap_base &&other) +heap_base &heap_base::operator=(heap_base &&other) noexcept { cnt = std::move(other.cnt); flavour_ = std::move(other.flavour_); @@ -311,7 +311,7 @@ s.flush(); } } - s.stop_received(); + s.stop(); return s.descriptors; } diff -Nru spead2-1.10.0/src/recv_inproc.cpp spead2-2.1.0/src/recv_inproc.cpp --- spead2-1.10.0/src/recv_inproc.cpp 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/src/recv_inproc.cpp 2019-04-30 16:10:58.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2018 SKA South Africa +/* Copyright 2018-2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -36,7 +36,7 @@ std::shared_ptr queue) : reader(owner), queue(std::move(queue)), - data_sem_wrapper(wrap_fd(get_io_service(), + data_sem_wrapper(wrap_fd(owner.get_io_service(), this->queue->buffer.get_data_sem().get_fd())) { enqueue(); @@ -49,7 +49,7 @@ std::size_t size = decode_packet(header, packet.data.get(), packet.size); if (size == packet.size) { - get_stream_base().add_packet(state, header); + state.add_packet(header); } else if (size != 0) { @@ -61,29 +61,35 @@ const boost::system::error_code &error, std::size_t bytes_transferred) { - if (get_stream_base().is_stopped()) + stream_base::add_packet_state state(get_stream_base()); + if (!error) { - log_info("inproc reader: discarding packet received after stream stopped"); - } - else - { - stream_base::add_packet_state state(get_stream_base()); - try + if (state.is_stopped()) { - inproc_queue::packet packet = queue->buffer.try_pop(); - process_one_packet(state, packet); - /* TODO: could grab a batch of packets to amortise costs */ + log_info("inproc reader: discarding packet received after stream stopped"); } - catch (ringbuffer_stopped) + else { - get_stream_base().stop_received(); - } - catch (ringbuffer_empty) - { - // spurious wakeup - no action needed + try + { + inproc_queue::packet packet = queue->buffer.try_pop(); + process_one_packet(state, packet); + /* TODO: could grab a batch of packets to amortise costs */ + } + catch (ringbuffer_stopped &) + { + state.stop(); + } + catch (ringbuffer_empty &) + { + // spurious wakeup - no action needed + } } } - if (!get_stream_base().is_stopped()) + else if (error != boost::asio::error::operation_aborted) + log_warning("Error in inproc receiver: %1%", error.message()); + + if (!state.is_stopped()) enqueue(); else { @@ -97,7 +103,7 @@ using namespace std::placeholders; data_sem_wrapper.async_read_some( boost::asio::null_buffers(), - get_stream().get_strand().wrap(std::bind(&inproc_reader::packet_handler, this, _1, _2))); + std::bind(&inproc_reader::packet_handler, this, _1, _2)); } void inproc_reader::stop() diff -Nru spead2-1.10.0/src/recv_live_heap.cpp spead2-2.1.0/src/recv_live_heap.cpp --- spead2-1.10.0/src/recv_live_heap.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/recv_live_heap.cpp 2019-03-06 09:53:45.000000000 +0000 @@ -44,7 +44,6 @@ } void live_heap::payload_reserve(std::size_t size, bool exact, const packet_header &packet, - const memcpy_function &memcpy, memory_allocator &allocator) { if (size > payload_reserved) @@ -55,8 +54,8 @@ } memory_allocator::pointer new_payload; new_payload = allocator.allocate(size, (void *) &packet); - if (payload) - memcpy(new_payload.get(), payload.get(), payload_reserved); + if (payload && new_payload) + std::memcpy(new_payload.get(), payload.get(), payload_reserved); payload = std::move(new_payload); payload_reserved = size; } @@ -151,7 +150,8 @@ } } -bool live_heap::add_packet(const packet_header &packet, const memcpy_function &memcpy, +bool live_heap::add_packet(const packet_header &packet, + const packet_memcpy_function &packet_memcpy, memory_allocator &allocator) { /* It's important that these initial checks can't fail for a @@ -195,23 +195,20 @@ { heap_length = packet.heap_length; min_length = std::max(min_length, heap_length); - payload_reserve(min_length, true, packet, memcpy, allocator); + payload_reserve(min_length, true, packet, allocator); } } else { min_length = std::max(min_length, packet.payload_offset + packet.payload_length); - payload_reserve(min_length, false, packet, memcpy, allocator); + payload_reserve(min_length, false, packet, allocator); } add_pointers(packet.n_items, packet.pointers); if (packet.payload_length > 0) { - // Note: this is the memcpy function pointer passed in, not std::memcpy - memcpy(payload.get() + packet.payload_offset, - packet.payload, - packet.payload_length); + packet_memcpy(payload, packet); received_length += packet.payload_length; } log_debug("packet with %d bytes of payload at offset %d added to heap %d", diff -Nru spead2-1.10.0/src/recv_mem.cpp spead2-2.1.0/src/recv_mem.cpp --- spead2-1.10.0/src/recv_mem.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/recv_mem.cpp 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -35,10 +35,10 @@ : reader(owner), ptr(ptr), length(length) { assert(ptr != nullptr); - get_stream().get_strand().post([this] { + get_io_service().post([this] { mem_to_stream(get_stream_base(), this->ptr, this->length); // There will be no more data, so we can stop the stream immediately. - get_stream_base().stop_received(); + get_stream_base().stop(); stopped(); }); } diff -Nru spead2-1.10.0/src/recv_netmap.cpp spead2-2.1.0/src/recv_netmap.cpp --- spead2-1.10.0/src/recv_netmap.cpp 2018-12-06 11:40:49.000000000 +0000 +++ spead2-2.1.0/src/recv_netmap.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,158 +0,0 @@ -/* Copyright 2015, 2018 SKA South Africa - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more - * details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -/** - * @file - */ - -#include - -#if SPEAD2_USE_NETMAP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace spead2 -{ -namespace recv -{ - -namespace detail -{ - -void nm_desc_destructor::operator()(nm_desc *d) const -{ - // We wrap the fd in an asio handle, which takes care of closing it. - // To prevent nm_close from closing it too, we nullify it here. - d->fd = -1; - int status = nm_close(d); - if (status != 0) - { - std::error_code code(status, std::system_category()); - log_warning("Failed to close the netmap fd: %1% (%2%)", code.value(), code.message()); - } -} - -} // namespace detail - -netmap_udp_reader::netmap_udp_reader(stream &owner, const std::string &device, std::uint16_t port) - : reader(owner), handle(get_io_service()), - desc(nm_open(("netmap:" + device + "*").c_str(), NULL, 0, NULL)), - port(port) -{ - if (!desc) - throw std::system_error(errno, std::system_category()); - handle.assign(desc->fd); - enqueue_receive(); -} - -void netmap_udp_reader::packet_handler(const boost::system::error_code &error) -{ - if (!error) - { - stream_base::add_packet_state state(get_stream_base()); - for (int ri = desc->first_rx_ring; ri <= desc->last_rx_ring; ri++) - { - netmap_ring *ring = NETMAP_RXRING(desc->nifp, ri); - ring->flags |= NR_FORWARD | NR_TIMESTAMP; - for (unsigned int i = ring->cur; i != ring->tail; i = nm_ring_next(ring, i)) - { - auto &slot = ring->slot[i]; - bool used = false; - // Skip even trying to process packets in the host ring - if (ri != desc->req.nr_rx_rings - && !(slot.flags & NS_MOREFRAG)) - { - const std::uint8_t *data = (const std::uint8_t *) NETMAP_BUF(ring, slot.buf_idx); - packet_buffer payload; - try - { - ethernet_frame eth(const_cast(data), slot.len); - if (eth.ethertype() == ipv4_packet::ethertype) - { - ipv4_packet ipv4 = eth.payload_ipv4(); - if (ipv4.version() == 4 - && ipv4.protocol() == udp_packet::protocol - && !ipv4.is_fragment()) - { - udp_packet udp = ipv4.payload_udp(); - if (udp.destination_port() == port) - { - used = true; - payload = udp.payload(); - } - } - } - } - catch (std::length_error) - { - // just pass them to the host stack - } - if (used) - { - packet_header packet; - std::size_t size = decode_packet(packet, payload.data(), payload.size()); - if (size == payload.size()) - { - get_stream_base().add_packet(state, packet); - if (get_stream_base().is_stopped()) - log_debug("netmap_udp_reader: end of stream detected"); - } - else if (size != 0) - { - log_info("discarding packet due to size mismatch (%1% != %2%) flags = %3%", - size, payload.size(), slot.flags); - } - } - } - if (!used) - slot.flags |= NS_FORWARD; - } - ring->cur = ring->head = ring->tail; - } - } - else - log_warning("error in netmap receive: %1% (%2%)", error.value(), error.message()); - - if (get_stream_base().is_stopped()) - stopped(); - else - enqueue_receive(); -} - -void netmap_udp_reader::enqueue_receive() -{ - using namespace std::placeholders; - handle.async_read_some( - boost::asio::null_buffers(), - get_stream().get_strand().wrap(std::bind(&netmap_udp_reader::packet_handler, this, _1))); -} - -void netmap_udp_reader::stop() -{ - handle.cancel(); -} - -} // namespace recv -} // namespace spead2 - -#endif // SPEAD2_USE_NETMAP diff -Nru spead2-1.10.0/src/recv_reader.cpp spead2-2.1.0/src/recv_reader.cpp --- spead2-1.10.0/src/recv_reader.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/recv_reader.cpp 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -28,7 +28,10 @@ void reader::stopped() { - stopped_promise.set_value(); + // Schedule it to run later so that at the time it occurs there are no + // further references to *this. + stream *owner_ptr = &owner; + get_io_service().post([owner_ptr] { owner_ptr->readers_stopped.put(); }); } bool reader::lossy() const @@ -38,7 +41,7 @@ boost::asio::io_service &reader::get_io_service() { - return owner.get_strand().get_io_service(); + return owner.get_io_service(); } stream_base &reader::get_stream_base() const @@ -46,10 +49,5 @@ return owner; } -void reader::join() -{ - stopped_promise.get_future().get(); -} - } // namespace recv } // namespace spead2 diff -Nru spead2-1.10.0/src/recv_stream.cpp spead2-2.1.0/src/recv_stream.cpp --- spead2-1.10.0/src/recv_stream.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/recv_stream.cpp 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017, 2018 SKA South Africa +/* Copyright 2015, 2017-2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -27,6 +27,7 @@ #include #include #include +#include #define INVALID_ENTRY ((queue_entry *) -1) @@ -81,6 +82,12 @@ return shift; } +#define SPEAD2_ADAPT_MEMCPY(func, capture) \ + (packet_memcpy_function([capture](const spead2::memory_allocator::pointer &allocation, const packet_header &packet) \ + { \ + func(allocation.get() + packet.payload_offset, packet.payload, packet.payload_length); \ + })) + stream_base::stream_base(bug_compat_mask bug_compat, std::size_t max_heaps) : queue_storage(new storage_type[max_heaps]), bucket_count(compute_bucket_count(max_heaps)), @@ -88,6 +95,7 @@ buckets(new queue_entry *[bucket_count]), head(0), max_heaps(max_heaps), bug_compat(bug_compat), + memcpy(SPEAD2_ADAPT_MEMCPY(std::memcpy, )), allocator(std::make_shared()) { if (max_heaps == 0) @@ -143,24 +151,34 @@ void stream_base::set_memory_allocator(std::shared_ptr allocator) { - std::lock_guard lock(allocator_mutex); + std::lock_guard lock(config_mutex); this->allocator = std::move(allocator); } +void stream_base::set_memcpy(packet_memcpy_function memcpy) +{ + std::lock_guard lock(config_mutex); + this->memcpy = memcpy; +} + void stream_base::set_memcpy(memcpy_function memcpy) { - this->memcpy.store(memcpy, std::memory_order_relaxed); + set_memcpy(SPEAD2_ADAPT_MEMCPY(memcpy, memcpy)); } void stream_base::set_memcpy(memcpy_function_id id) { + /* We adapt each case to the packet_memcpy signature rather than using the + * generic wrapping in the memcpy_function overload. This ensures that + * there is only one level of indirect function call instead of two. + */ switch (id) { case MEMCPY_STD: - set_memcpy(std::memcpy); + set_memcpy(SPEAD2_ADAPT_MEMCPY(std::memcpy, )); break; case MEMCPY_NONTEMPORAL: - set_memcpy(spead2::memcpy_nontemporal); + set_memcpy(SPEAD2_ADAPT_MEMCPY(spead2::memcpy_nontemporal, )); break; default: throw std::invalid_argument("Unknown memcpy function"); @@ -169,25 +187,43 @@ void stream_base::set_stop_on_stop_item(bool stop) { + std::lock_guard lock(config_mutex); stop_on_stop_item = stop; } bool stream_base::get_stop_on_stop_item() const { - return stop_on_stop_item.load(); + std::lock_guard lock(config_mutex); + return stop_on_stop_item; +} + +void stream_base::set_allow_unsized_heaps(bool allow) +{ + std::lock_guard lock(config_mutex); + allow_unsized_heaps = allow; +} + +bool stream_base::get_allow_unsized_heaps() const +{ + std::lock_guard lock(config_mutex); + return allow_unsized_heaps; } stream_base::add_packet_state::add_packet_state(stream_base &owner) - : owner(owner), memcpy(owner.memcpy.load()), - stop_on_stop_item(owner.stop_on_stop_item.load()) + : owner(owner), lock(owner.queue_mutex) { - std::lock_guard lock(owner.allocator_mutex); + std::lock_guard config_lock(owner.config_mutex); allocator = owner.allocator; + memcpy = owner.memcpy; + stop_on_stop_item = owner.stop_on_stop_item; + allow_unsized_heaps = owner.allow_unsized_heaps; } stream_base::add_packet_state::~add_packet_state() { std::lock_guard stats_lock(owner.stats_mutex); + if (!packets && is_stopped()) + return; // Stream was stopped before we could do anything - don't count as a batch owner.stats.packets += packets; owner.stats.batches++; owner.stats.heaps += complete_heaps + incomplete_heaps_evicted; @@ -200,6 +236,13 @@ bool stream_base::add_packet(add_packet_state &state, const packet_header &packet) { assert(!stopped); + state.packets++; + if (packet.heap_length < 0 && !state.allow_unsized_heaps) + { + log_info("packet rejected because it has no HEAP_LEN"); + return false; + } + // Look for matching heap. queue_entry *entry = NULL; s_item_pointer_t heap_cnt = packet.heap_cnt; @@ -260,14 +303,13 @@ h->~live_heap(); } } - state.packets++; if (end_of_stream) stop_received(); return result; } -void stream_base::flush() +void stream_base::flush_unlocked() { std::size_t n_flushed = 0; for (std::size_t i = 0; i < max_heaps; i++) @@ -288,54 +330,82 @@ stats.incomplete_heaps_flushed += n_flushed; } -void stream_base::stop_received() +void stream_base::flush() { - stopped = true; - flush(); + std::lock_guard lock(queue_mutex); + flush_unlocked(); } +void stream_base::stop_unlocked() +{ + if (!stopped) + stop_received(); +} -stream::stream(io_service_ref io_service, bug_compat_mask bug_compat, std::size_t max_heaps) - : stream_base(bug_compat, max_heaps), - thread_pool_holder(std::move(io_service).get_shared_thread_pool()), - strand(*io_service), lossy(false) +void stream_base::stop() { + std::lock_guard lock(queue_mutex); + stop_unlocked(); } -stream_stats stream::get_stats() const +void stream_base::stop_received() +{ + assert(!stopped); + stopped = true; + flush_unlocked(); +} + +stream_stats stream_base::get_stats() const { std::lock_guard stats_lock(stats_mutex); stream_stats ret = stats; return ret; } + +stream::stream(io_service_ref io_service, bug_compat_mask bug_compat, std::size_t max_heaps) + : stream_base(bug_compat, max_heaps), + thread_pool_holder(std::move(io_service).get_shared_thread_pool()), + io_service(*io_service) +{ +} + void stream::stop_received() { - // Check for already stopped, so that readers are stopped exactly once - if (!is_stopped()) - { - stream_base::stop_received(); - for (const auto &reader : readers) - reader->stop(); - } + stream_base::stop_received(); + std::lock_guard lock(reader_mutex); + for (const auto &reader : readers) + reader->stop(); } void stream::stop_impl() { - run_in_strand([this] { stop_received(); }); + stream_base::stop(); - /* Block until all readers have entered their final completion handler. - * Note that this cannot conflict with a previously issued emplace_reader, - * because its emplace_reader_callback either happens-before the - * stop_received call above or sees that the stream has been stopped and - * does not touch the readers list. - */ - for (const auto &r : readers) - r->join(); + std::size_t n_readers; + { + std::lock_guard lock(reader_mutex); + /* Prevent any further calls to emplace_reader from doing anything, so + * that n_readers will remain accurate. + */ + stop_readers = true; + n_readers = readers.size(); + } - // Destroy the readers with the strand held, to ensure that the - // completion handlers have actually returned. - run_in_strand([this] { readers.clear(); }); + // Wait until all readers have wound up all their completion handlers + while (n_readers > 0) + { + semaphore_get(readers_stopped); + n_readers--; + } + + { + /* This lock is not strictly needed since no other thread can touch + * readers any more, but is harmless. + */ + std::lock_guard lock(reader_mutex); + readers.clear(); + } } void stream::stop() @@ -343,6 +413,12 @@ std::call_once(stop_once, [this] { stop_impl(); }); } +bool stream::is_lossy() const +{ + std::lock_guard lock(reader_mutex); + return lossy; +} + stream::~stream() { stop(); @@ -352,13 +428,13 @@ const std::uint8_t *mem_to_stream(stream_base &s, const std::uint8_t *ptr, std::size_t length) { stream_base::add_packet_state state(s); - while (length > 0 && !s.is_stopped()) + while (length > 0 && !state.is_stopped()) { packet_header packet; std::size_t size = decode_packet(packet, ptr, length); if (size > 0) { - s.add_packet(state, packet); + state.add_packet(packet); ptr += size; length -= size; } diff -Nru spead2-1.10.0/src/recv_tcp.cpp spead2-2.1.0/src/recv_tcp.cpp --- spead2-1.10.0/src/recv_tcp.cpp 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/src/recv_tcp.cpp 2019-10-11 09:23:12.000000000 +0000 @@ -51,17 +51,16 @@ std::size_t max_size, std::size_t buffer_size) : reader(owner), acceptor(std::move(acceptor)), - peer(this->acceptor.get_io_service()), + peer(get_socket_io_service(this->acceptor)), max_size(max_size), buffer(new std::uint8_t[max_size * pkts_per_buffer]), head(buffer.get()), tail(buffer.get()) { - assert(&this->acceptor.get_io_service() == &get_io_service()); + assert(socket_uses_io_service(this->acceptor, get_io_service())); set_socket_recv_buffer_size(this->acceptor, buffer_size); this->acceptor.async_accept(peer, - get_stream().get_strand().wrap( - std::bind(&tcp_reader::accept_handler, this, std::placeholders::_1))); + std::bind(&tcp_reader::accept_handler, this, std::placeholders::_1)); } tcp_reader::tcp_reader( @@ -71,7 +70,7 @@ std::size_t buffer_size) : tcp_reader( owner, - boost::asio::ip::tcp::acceptor(owner.get_strand().get_io_service(), endpoint), + boost::asio::ip::tcp::acceptor(owner.get_io_service(), endpoint), max_size, buffer_size) { } @@ -88,17 +87,19 @@ const boost::system::error_code &error, std::size_t bytes_transferred) { + stream_base::add_packet_state state(get_stream_base()); + bool read_more = false; if (!error) { - if (get_stream_base().is_stopped()) + if (state.is_stopped()) log_info("TCP reader: discarding packet received after stream stopped"); else - read_more = process_buffer(bytes_transferred); + read_more = process_buffer(state, bytes_transferred); } else if (error == boost::asio::error::eof) { - get_stream_base().stop_received(); + state.stop(); read_more = false; } else if (error != boost::asio::error::operation_aborted) @@ -113,7 +114,7 @@ } } -bool tcp_reader::parse_packet() +bool tcp_reader::parse_packet(stream_base::add_packet_state &state) { assert(pkt_size > 0); assert(tail - head >= pkt_size); @@ -127,10 +128,8 @@ std::size_t size = decode_packet(packet, head, pkt_size); if (size == pkt_size) { - stream_base &s = get_stream_base(); - stream_base::add_packet_state state(s); - s.add_packet(state, packet); - if (s.is_stopped()) + state.add_packet(packet); + if (state.is_stopped()) { log_debug("TCP reader: end of stream detected"); return true; @@ -139,7 +138,7 @@ return false; } -bool tcp_reader::process_buffer(const std::size_t bytes_recv) +bool tcp_reader::process_buffer(stream_base::add_packet_state &state, const std::size_t bytes_recv) { tail += bytes_recv; while (tail > head) @@ -152,7 +151,7 @@ continue; if (std::size_t(tail - head) < pkt_size) return true; - if (parse_packet()) + if (parse_packet(state)) return false; } return true; @@ -217,6 +216,13 @@ void tcp_reader::accept_handler(const boost::system::error_code &error) { + /* We need to hold the stream's queue_mutex, because that guards access + * to the sockets. This is a heavy-weight way to do it, but since it + * only happens once per connection it is probably not worth trying to + * add a lighter-weight interface to @c stream. + */ + stream_base::add_packet_state state(get_stream_base()); + acceptor.close(); if (!error) enqueue_receive(); @@ -248,9 +254,7 @@ peer.async_receive( boost::asio::buffer(tail, bufsize - (tail - buf)), - get_stream().get_strand().wrap( - std::bind(&tcp_reader::packet_handler, - this, _1, _2))); + std::bind(&tcp_reader::packet_handler, this, _1, _2)); } void tcp_reader::stop() diff -Nru spead2-1.10.0/src/recv_udp_base.cpp spead2-2.1.0/src/recv_udp_base.cpp --- spead2-1.10.0/src/recv_udp_base.cpp 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/src/recv_udp_base.cpp 2019-11-13 09:34:53.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016 SKA South Africa +/* Copyright 2016, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -45,8 +45,8 @@ std::size_t size = decode_packet(packet, data, length); if (size == length) { - get_stream_base().add_packet(state, packet); - if (get_stream_base().is_stopped()) + state.add_packet(packet); + if (state.is_stopped()) { log_debug("UDP reader: end of stream detected"); stopped = true; diff -Nru spead2-1.10.0/src/recv_udp.cpp spead2-2.1.0/src/recv_udp.cpp --- spead2-1.10.0/src/recv_udp.cpp 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/src/recv_udp.cpp 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -49,28 +49,6 @@ constexpr std::size_t udp_reader::default_buffer_size; -#if SPEAD2_USE_RECVMMSG -static boost::asio::ip::udp::socket duplicate_socket( - boost::asio::ip::udp::socket &socket) -{ - int fd = socket.native_handle(); - int fd2 = dup(fd); - if (fd2 < 0) - throw_errno("dup failed"); - try - { - boost::asio::ip::udp::socket socket2( - socket.get_io_service(), socket.local_endpoint().protocol(), fd2); - return socket2; - } - catch (std::exception) - { - close(fd2); - throw; - } -} -#endif - static boost::asio::ip::udp::socket bind_socket( boost::asio::ip::udp::socket &&socket, const boost::asio::ip::udp::endpoint &endpoint, @@ -87,13 +65,12 @@ std::size_t max_size) : udp_reader_base(owner), socket(std::move(socket)), max_size(max_size), #if SPEAD2_USE_RECVMMSG - socket2(socket.get_io_service()), buffer(mmsg_count), iov(mmsg_count), msgvec(mmsg_count) #else buffer(new std::uint8_t[max_size + 1]) #endif { - assert(&this->socket.get_io_service() == &get_io_service()); + assert(socket_uses_io_service(this->socket, get_io_service())); #if SPEAD2_USE_RECVMMSG for (std::size_t i = 0; i < mmsg_count; i++) { @@ -107,9 +84,6 @@ } #endif -#if SPEAD2_USE_RECVMMSG - socket2 = duplicate_socket(this->socket); -#endif enqueue_receive(); } @@ -123,21 +97,29 @@ { } -static boost::asio::ip::udp::socket make_multicast_v4_socket( +static boost::asio::ip::udp::socket make_bound_v4_socket( boost::asio::io_service &io_service, const boost::asio::ip::udp::endpoint &endpoint, std::size_t buffer_size, const boost::asio::ip::address &interface_address) { - if (!endpoint.address().is_v4() || !endpoint.address().is_multicast()) - throw std::invalid_argument("endpoint is not an IPv4 multicast address"); if (!interface_address.is_v4()) throw std::invalid_argument("interface address is not an IPv4 address"); - boost::asio::ip::udp::socket socket(io_service, endpoint.protocol()); - socket.set_option(boost::asio::socket_base::reuse_address(true)); - socket.set_option(boost::asio::ip::multicast::join_group( - endpoint.address().to_v4(), interface_address.to_v4())); - return bind_socket(std::move(socket), endpoint, buffer_size); + auto ep = endpoint; + if (ep.address().is_unspecified()) + ep.address(interface_address); + if (!ep.address().is_v4()) + throw std::invalid_argument("endpoint is not an IPv4 address"); + if (!ep.address().is_multicast() && ep.address() != interface_address) + throw std::invalid_argument("endpoint is not multicast and does not match interface address"); + boost::asio::ip::udp::socket socket(io_service, ep.protocol()); + if (ep.address().is_multicast()) + { + socket.set_option(boost::asio::socket_base::reuse_address(true)); + socket.set_option(boost::asio::ip::multicast::join_group( + ep.address().to_v4(), interface_address.to_v4())); + } + return bind_socket(std::move(socket), ep, buffer_size); } static boost::asio::ip::udp::socket make_multicast_v6_socket( @@ -176,7 +158,7 @@ std::size_t buffer_size) : udp_reader( owner, - make_socket(owner.get_strand().get_io_service(), endpoint, buffer_size), + make_socket(owner.get_io_service(), endpoint, buffer_size), max_size) { } @@ -189,8 +171,8 @@ const boost::asio::ip::address &interface_address) : udp_reader( owner, - make_multicast_v4_socket(owner.get_strand().get_io_service(), - endpoint, buffer_size, interface_address), + make_bound_v4_socket(owner.get_io_service(), + endpoint, buffer_size, interface_address), max_size) { } @@ -203,7 +185,7 @@ unsigned int interface_index) : udp_reader( owner, - make_multicast_v6_socket(owner.get_strand().get_io_service(), + make_multicast_v6_socket(owner.get_io_service(), endpoint, buffer_size, interface_index), max_size) { @@ -213,17 +195,17 @@ const boost::system::error_code &error, std::size_t bytes_transferred) { + stream_base::add_packet_state state(get_stream_base()); if (!error) { - stream_base::add_packet_state state(get_stream_base()); - if (get_stream_base().is_stopped()) + if (state.is_stopped()) { log_info("UDP reader: discarding packet received after stream stopped"); } else { #if SPEAD2_USE_RECVMMSG - int received = recvmmsg(socket2.native_handle(), msgvec.data(), msgvec.size(), + int received = recvmmsg(socket.native_handle(), msgvec.data(), msgvec.size(), MSG_DONTWAIT, nullptr); log_debug("recvmmsg returned %1%", received); if (received == -1 && errno != EAGAIN && errno != EWOULDBLOCK) @@ -246,15 +228,12 @@ else if (error != boost::asio::error::operation_aborted) log_warning("Error in UDP receiver: %1%", error.message()); - if (!get_stream_base().is_stopped()) + if (!state.is_stopped()) { enqueue_receive(); } else { -#if SPEAD2_USE_RECVMMSG - socket2.close(); -#endif stopped(); } } @@ -269,7 +248,7 @@ boost::asio::buffer(buffer.get(), max_size + 1), #endif endpoint, - get_stream().get_strand().wrap(std::bind(&udp_reader::packet_handler, this, _1, _2))); + std::bind(&udp_reader::packet_handler, this, _1, _2)); } void udp_reader::stop() @@ -316,7 +295,7 @@ { ibv_comp_vector = boost::lexical_cast(comp_vector); } - catch (boost::bad_lexical_cast) + catch (boost::bad_lexical_cast &) { log_warning("SPEAD2_IBV_COMP_VECTOR is not a valid integer, ignoring"); } @@ -332,7 +311,7 @@ std::size_t max_size, std::size_t buffer_size) { - if (endpoint.address().is_v4() && endpoint.address().is_multicast()) + if (endpoint.address().is_v4()) { std::call_once(ibv_once, init_ibv_override); #if SPEAD2_USE_IBV @@ -340,8 +319,8 @@ { log_info("Overriding reader for %1%:%2% to use ibverbs", endpoint.address().to_string(), endpoint.port()); - return std::unique_ptr(new udp_ibv_reader( - owner, endpoint, ibv_interface, max_size, buffer_size, ibv_comp_vector)); + return reader_factory::make_reader( + owner, endpoint, ibv_interface, max_size, buffer_size, ibv_comp_vector); } #endif } @@ -355,7 +334,7 @@ std::size_t buffer_size, const boost::asio::ip::address &interface_address) { - if (endpoint.address().is_v4() && endpoint.address().is_multicast()) + if (endpoint.address().is_v4()) { std::call_once(ibv_once, init_ibv_override); #if SPEAD2_USE_IBV @@ -363,8 +342,8 @@ { log_info("Overriding reader for %1%:%2% to use ibverbs", endpoint.address().to_string(), endpoint.port()); - return std::unique_ptr(new udp_ibv_reader( - owner, endpoint, interface_address, max_size, buffer_size, ibv_comp_vector)); + return reader_factory::make_reader( + owner, endpoint, interface_address, max_size, buffer_size, ibv_comp_vector); } #endif } diff -Nru spead2-1.10.0/src/recv_udp_ibv.cpp spead2-2.1.0/src/recv_udp_ibv.cpp --- spead2-1.10.0/src/recv_udp_ibv.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/recv_udp_ibv.cpp 2020-02-12 09:25:32.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016 SKA South Africa +/* Copyright 2016-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -46,12 +47,108 @@ namespace recv { -constexpr std::size_t udp_ibv_reader::default_buffer_size; -constexpr int udp_ibv_reader::default_max_poll; +namespace detail +{ + +constexpr std::size_t udp_ibv_reader_core::default_buffer_size; +constexpr int udp_ibv_reader_core::default_max_poll; static constexpr int header_length = ethernet_frame::min_size + ipv4_packet::min_size + udp_packet::min_size; -ibv_qp_t udp_ibv_reader::create_qp( +static spead2::rdma_cm_id_t make_cm_id(const rdma_event_channel_t &event_channel, + const boost::asio::ip::address &interface_address) +{ + if (!interface_address.is_v4()) + throw std::invalid_argument("interface address is not an IPv4 address"); + rdma_cm_id_t cm_id(event_channel, nullptr, RDMA_PS_UDP); + cm_id.bind_addr(interface_address); + return cm_id; +} + +udp_ibv_reader_core::udp_ibv_reader_core( + stream &owner, + const std::vector &endpoints, + const boost::asio::ip::address &interface_address, + std::size_t max_size, + int comp_vector, + int max_poll) + : udp_reader_base(owner), + join_socket(owner.get_io_service(), boost::asio::ip::udp::v4()), + comp_channel_wrapper(owner.get_io_service()), + max_size(max_size), + max_poll(max_poll), + stop_poll(false) +{ + for (const auto &endpoint : endpoints) + if (!endpoint.address().is_unspecified() && !endpoint.address().is_v4()) + { + std::ostringstream msg; + msg << "endpoint " << endpoint << " is not an IPv4 address"; + throw std::invalid_argument(msg.str()); + } + if (max_poll <= 0) + throw std::invalid_argument("max_poll must be non-negative"); + + cm_id = make_cm_id(event_channel, interface_address); + pd = ibv_pd_t(cm_id); + if (comp_vector >= 0) + { + comp_channel = ibv_comp_channel_t(cm_id); + comp_channel_wrapper = comp_channel.wrap(get_io_service()); + } +} + +void udp_ibv_reader_core::join_groups( + const std::vector &endpoints, + const boost::asio::ip::address &interface_address) +{ + join_socket.set_option(boost::asio::socket_base::reuse_address(true)); + for (const auto &endpoint : endpoints) + if (endpoint.address().is_multicast()) + { + join_socket.set_option(boost::asio::ip::multicast::join_group( + endpoint.address().to_v4(), interface_address.to_v4())); + } +} + +void udp_ibv_reader_core::stop() +{ + if (comp_channel) + comp_channel_wrapper.close(); + else + stop_poll = true; +} + +} // namespace detail + +static std::size_t compute_n_slots(const rdma_cm_id_t &cm_id, std::size_t buffer_size, + std::size_t max_raw_size) +{ + bool reduced = false; + ibv_device_attr attr = cm_id.query_device(); + if (attr.max_mr_size < max_raw_size) + throw std::invalid_argument("Packet size is larger than biggest MR supported by device"); + if (attr.max_mr_size < buffer_size) + { + buffer_size = attr.max_mr_size; + reduced = true; + } + + std::size_t n_slots = std::max(std::size_t(1), buffer_size / max_raw_size); + std::size_t hw_slots = std::min(attr.max_qp_wr, attr.max_cqe); + if (hw_slots == 0) + throw std::invalid_argument("This device does not have a usable verbs implementation"); + if (hw_slots < n_slots) + { + n_slots = hw_slots; + reduced = true; + } + if (reduced) + log_warning("Reducing buffer to %1% to accommodate device limits", n_slots * max_raw_size); + return n_slots; +} + +static ibv_qp_t create_qp( const ibv_pd_t &pd, const ibv_cq_t &send_cq, const ibv_cq_t &recv_cq, std::size_t n_slots) { ibv_qp_init_attr attr; @@ -66,50 +163,7 @@ return ibv_qp_t(pd, &attr); } -ibv_flow_t udp_ibv_reader::create_flow( - const ibv_qp_t &qp, const boost::asio::ip::udp::endpoint &endpoint, int port_num) -{ - struct - { - ibv_flow_attr attr; - ibv_flow_spec_eth eth; - ibv_flow_spec_ipv4 ip; - ibv_flow_spec_tcp_udp udp; - } __attribute__((packed)) flow_rule; - memset(&flow_rule, 0, sizeof(flow_rule)); - - flow_rule.attr.type = IBV_FLOW_ATTR_NORMAL; - flow_rule.attr.priority = 0; - flow_rule.attr.size = sizeof(flow_rule); - flow_rule.attr.num_of_specs = 3; - flow_rule.attr.port = port_num; - - /* At least the ConnectX-3 cards seem to require an Ethernet match. We - * thus have to construct the Ethernet multicast address corresponding to - * the IP multicast address from RFC 7042. - */ - flow_rule.eth.type = IBV_FLOW_SPEC_ETH; - flow_rule.eth.size = sizeof(flow_rule.eth); - mac_address dst_mac = multicast_mac(endpoint.address()); - std::memcpy(&flow_rule.eth.val.dst_mac, &dst_mac, sizeof(dst_mac)); - // Set all 1's mask - std::memset(&flow_rule.eth.mask.dst_mac, 0xFF, sizeof(flow_rule.eth.mask.dst_mac)); - - flow_rule.ip.type = IBV_FLOW_SPEC_IPV4; - flow_rule.ip.size = sizeof(flow_rule.ip); - auto bytes = endpoint.address().to_v4().to_bytes(); // big-endian address - std::memcpy(&flow_rule.ip.val.dst_ip, &bytes, sizeof(bytes)); - std::memset(&flow_rule.ip.mask.dst_ip, 0xFF, sizeof(flow_rule.ip.mask.dst_ip)); - - flow_rule.udp.type = IBV_FLOW_SPEC_UDP; - flow_rule.udp.size = sizeof(flow_rule.udp); - flow_rule.udp.val.dst_port = htobe16(endpoint.port()); - flow_rule.udp.mask.dst_port = 0xFFFF; - - return ibv_flow_t(qp, &flow_rule.attr); -} - -int udp_ibv_reader::poll_once(stream_base::add_packet_state &state) +udp_ibv_reader::poll_result udp_ibv_reader::poll_once(stream_base::add_packet_state &state) { /* Number of work requests to queue at a time. This is a balance between * not calling post_recv too often (it takes a lock) and not starving the @@ -139,7 +193,7 @@ bool stopped = process_one_packet(state, payload.data(), payload.size(), max_size); if (stopped) - return -2; + return poll_result::stopped; } catch (packet_type_error &e) { @@ -168,92 +222,7 @@ tail->next = nullptr; qp.post_recv(head); } - return received; -} - -void udp_ibv_reader::packet_handler(const boost::system::error_code &error) -{ - if (!error) - { - if (get_stream_base().is_stopped()) - { - log_info("UDP reader: discarding packet received after stream stopped"); - } - else - { - stream_base::add_packet_state state(get_stream_base()); - if (comp_channel) - { - ibv_cq *event_cq; - void *event_context; - comp_channel.get_event(&event_cq, &event_context); - // TODO: defer acks until shutdown - recv_cq.ack_events(1); - } - for (int i = 0; i < max_poll; i++) - { - if (comp_channel) - { - if (i == max_poll - 1) - { - /* We need to call req_notify_cq *before* the last - * poll_once, because notifications are edge-triggered. - * If we did it the other way around, there is a race - * where a new packet can arrive after poll_once but - * before req_notify_cq, failing to trigger a - * notification. - */ - recv_cq.req_notify(false); - } - } - else if (stop_poll.load()) - break; - int received = poll_once(state); - if (received < 0) - break; - } - } - } - else if (error != boost::asio::error::operation_aborted) - log_warning("Error in UDP receiver: %1%", error.message()); - - if (!get_stream_base().is_stopped()) - { - enqueue_receive(); - } - else - stopped(); -} - -void udp_ibv_reader::enqueue_receive() -{ - using namespace std::placeholders; - if (comp_channel) - { - // Asynchronous mode - comp_channel_wrapper.async_read_some( - boost::asio::null_buffers(), - get_stream().get_strand().wrap(std::bind(&udp_ibv_reader::packet_handler, this, _1))); - } - else - { - // Polling mode - get_stream().get_strand().post(std::bind( - &udp_ibv_reader::packet_handler, this, boost::system::error_code())); - } -} - -udp_ibv_reader::udp_ibv_reader( - stream &owner, - const boost::asio::ip::udp::endpoint &endpoint, - const boost::asio::ip::address &interface_address, - std::size_t max_size, - std::size_t buffer_size, - int comp_vector, - int max_poll) - : udp_ibv_reader(owner, std::vector{endpoint}, - interface_address, max_size, buffer_size, comp_vector, max_poll) -{ + return poll_result::drained; } udp_ibv_reader::udp_ibv_reader( @@ -264,47 +233,23 @@ std::size_t buffer_size, int comp_vector, int max_poll) - : udp_reader_base(owner), - max_size(max_size), - n_slots(std::max(std::size_t(1), buffer_size / (max_size + header_length))), - max_poll(max_poll), - join_socket(owner.get_strand().get_io_service(), boost::asio::ip::udp::v4()), - comp_channel_wrapper(owner.get_strand().get_io_service()), - stop_poll(false) + : udp_ibv_reader_base( + owner, endpoints, interface_address, max_size, comp_vector, max_poll), + n_slots(compute_n_slots(cm_id, buffer_size, max_size + detail::header_length)) { - for (const auto &endpoint : endpoints) - if (!endpoint.address().is_v4() || !endpoint.address().is_multicast()) - { - std::ostringstream msg; - msg << "endpoint " << endpoint << " is not an IPv4 multicast address"; - throw std::invalid_argument(msg.str()); - } - if (!interface_address.is_v4()) - throw std::invalid_argument("interface address is not an IPv4 address"); - if (max_poll <= 0) - throw std::invalid_argument("max_poll must be positive"); // Re-compute buffer_size as a whole number of slots - const std::size_t max_raw_size = max_size + header_length; + const std::size_t max_raw_size = max_size + detail::header_length; buffer_size = n_slots * max_raw_size; - cm_id = rdma_cm_id_t(event_channel, nullptr, RDMA_PS_UDP); - cm_id.bind_addr(interface_address); - if (comp_vector >= 0) - { - comp_channel = ibv_comp_channel_t(cm_id); - comp_channel_wrapper = comp_channel.wrap(owner.get_strand().get_io_service()); recv_cq = ibv_cq_t(cm_id, n_slots, nullptr, comp_channel, comp_vector % cm_id->verbs->num_comp_vectors); - } else recv_cq = ibv_cq_t(cm_id, n_slots, nullptr); send_cq = ibv_cq_t(cm_id, 1, nullptr); - pd = ibv_pd_t(cm_id); qp = create_qp(pd, send_cq, recv_cq, n_slots); qp.modify(IBV_QPS_INIT, cm_id->port_num); - for (const auto &endpoint : endpoints) - flows.push_back(create_flow(qp, endpoint, cm_id->port_num)); + flows = create_flows(qp, endpoints, cm_id->port_num); std::shared_ptr allocator = std::make_shared(0, true); buffer = allocator->allocate(buffer_size, nullptr); @@ -313,33 +258,32 @@ wc.reset(new ibv_wc[n_slots]); for (std::size_t i = 0; i < n_slots; i++) { - std::memset(&slots[i], 0, sizeof(slots[i])); slots[i].sge.addr = (uintptr_t) &buffer[i * max_raw_size]; slots[i].sge.length = max_raw_size; slots[i].sge.lkey = mr->lkey; + std::memset(&slots[i].wr, 0, sizeof(slots[i].wr)); slots[i].wr.sg_list = &slots[i].sge; slots[i].wr.num_sge = 1; slots[i].wr.wr_id = i; qp.post_recv(&slots[i].wr); } - join_socket.set_option(boost::asio::socket_base::reuse_address(true)); - for (const auto &endpoint : endpoints) - join_socket.set_option(boost::asio::ip::multicast::join_group( - endpoint.address().to_v4(), interface_address.to_v4())); - - if (comp_channel) - recv_cq.req_notify(false); - enqueue_receive(); + enqueue_receive(true); qp.modify(IBV_QPS_RTR); + join_groups(endpoints, interface_address); } -void udp_ibv_reader::stop() +udp_ibv_reader::udp_ibv_reader( + stream &owner, + const boost::asio::ip::udp::endpoint &endpoint, + const boost::asio::ip::address &interface_address, + std::size_t max_size, + std::size_t buffer_size, + int comp_vector, + int max_poll) + : udp_ibv_reader(owner, std::vector{endpoint}, + interface_address, max_size, buffer_size, comp_vector, max_poll) { - if (comp_channel) - comp_channel_wrapper.close(); - else - stop_poll = true; } } // namespace recv diff -Nru spead2-1.10.0/src/recv_udp_ibv_mprq.cpp spead2-2.1.0/src/recv_udp_ibv_mprq.cpp --- spead2-1.10.0/src/recv_udp_ibv_mprq.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/recv_udp_ibv_mprq.cpp 2019-04-30 16:11:05.000000000 +0000 @@ -0,0 +1,260 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + */ + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include +#if SPEAD2_USE_IBV_MPRQ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ +namespace recv +{ + +static ibv_qp_t create_qp(const rdma_cm_id_t &cm_id, + const ibv_pd_t &pd, + const ibv_cq_t &cq, + const ibv_exp_rwq_ind_table_t &ind_table, + const ibv_exp_res_domain_t &res_domain) +{ + /* ConnectX-5 only seems to work with Toeplitz hashing. This code seems to + * work, but I don't really know what I'm doing so it might be horrible. + */ + uint8_t toeplitz_key[40] = {}; + ibv_exp_rx_hash_conf hash_conf; + memset(&hash_conf, 0, sizeof(hash_conf)); + hash_conf.rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ; + hash_conf.rx_hash_key_len = sizeof(toeplitz_key); + hash_conf.rx_hash_key = toeplitz_key; + hash_conf.rx_hash_fields_mask = 0; + hash_conf.rwq_ind_tbl = ind_table.get(); + + ibv_exp_qp_init_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.qp_type = IBV_QPT_RAW_PACKET; + attr.pd = pd.get(); + attr.res_domain = res_domain.get(); + attr.rx_hash_conf = &hash_conf; + attr.port_num = cm_id->port_num; + attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_RX_HASH + | IBV_EXP_QP_INIT_ATTR_PORT | IBV_EXP_QP_INIT_ATTR_RES_DOMAIN; + return ibv_qp_t(cm_id, &attr); +} + +void udp_ibv_mprq_reader::post_wr(std::size_t offset) +{ + ibv_sge sge; + memset(&sge, 0, sizeof(sge)); + sge.addr = (uintptr_t) &buffer[offset]; + sge.length = wqe_size; + sge.lkey = mr->lkey; + int status = wq_intf->recv_burst(wq.get(), &sge, 1); + if (status != 0) + throw_errno("recv_burst failed", status); +} + +udp_ibv_mprq_reader::poll_result udp_ibv_mprq_reader::poll_once(stream_base::add_packet_state &state) +{ + /* Bound the number of times to receive packets, to avoid live-locking the + * receive queue if we're getting packets as fast as we can process them. + */ + const int max_iter = 256; + for (int iter = 0; iter < max_iter; iter++) + { + uint32_t offset; + uint32_t flags; + int32_t len = cq_intf->poll_length_flags_mp_rq(recv_cq.get(), &offset, &flags); + if (len < 0) + { + // Error condition. + ibv_wc wc; + ibv_poll_cq(recv_cq.get(), 1, &wc); + log_warning("Work Request failed with code %1%", wc.status); + } + else if (len > 0) + { + const void *ptr = reinterpret_cast(buffer.get() + (wqe_start + offset)); + + // Sanity checks + try + { + packet_buffer payload = udp_from_ethernet(const_cast(ptr), len); + bool stopped = process_one_packet(state, + payload.data(), payload.size(), max_size); + if (stopped) + return poll_result::stopped; + } + catch (packet_type_error &e) + { + log_warning(e.what()); + } + catch (std::length_error &e) + { + log_warning(e.what()); + } + } + if (flags & IBV_EXP_CQ_RX_MULTI_PACKET_LAST_V1) + { + post_wr(wqe_start); + wqe_start += wqe_size; + if (wqe_start == buffer_size) + wqe_start = 0; + } + if (len == 0 && flags == 0) + return poll_result::drained; + } + return poll_result::partial; +} + +static int clamp(int x, int low, int high) +{ + return std::min(std::max(x, low), high); +} + +udp_ibv_mprq_reader::udp_ibv_mprq_reader( + stream &owner, + const std::vector &endpoints, + const boost::asio::ip::address &interface_address, + std::size_t max_size, + std::size_t buffer_size, + int comp_vector, + int max_poll) + : udp_ibv_reader_base( + owner, endpoints, interface_address, max_size, comp_vector, max_poll) +{ + ibv_exp_device_attr device_attr = cm_id.exp_query_device(); + if (!(device_attr.comp_mask & IBV_EXP_DEVICE_ATTR_MP_RQ) + || !(device_attr.mp_rq_caps.supported_qps & IBV_EXP_MP_RQ_SUP_TYPE_WQ_RQ)) + throw std::system_error(std::make_error_code(std::errc::not_supported), + "device does not support multi-packet receive queues"); + + ibv_exp_res_domain_init_attr res_domain_attr; + memset(&res_domain_attr, 0, sizeof(res_domain_attr)); + res_domain_attr.comp_mask = IBV_EXP_RES_DOMAIN_THREAD_MODEL | IBV_EXP_RES_DOMAIN_MSG_MODEL; + res_domain_attr.thread_model = IBV_EXP_THREAD_UNSAFE; + res_domain_attr.msg_model = IBV_EXP_MSG_HIGH_BW; + res_domain = ibv_exp_res_domain_t(cm_id, &res_domain_attr); + + // TODO: adjust stride parameters based on device info + ibv_exp_wq_init_attr wq_attr; + memset(&wq_attr, 0, sizeof(wq_attr)); + wq_attr.mp_rq.single_stride_log_num_of_bytes = + clamp(6, + device_attr.mp_rq_caps.min_single_stride_log_num_of_bytes, + device_attr.mp_rq_caps.max_single_stride_log_num_of_bytes); // 64 bytes per stride + wq_attr.mp_rq.single_wqe_log_num_of_strides = + clamp(20 - wq_attr.mp_rq.single_stride_log_num_of_bytes, + device_attr.mp_rq_caps.min_single_wqe_log_num_of_strides, + device_attr.mp_rq_caps.max_single_wqe_log_num_of_strides); // 1MB per WQE + int log_wqe_size = wq_attr.mp_rq.single_stride_log_num_of_bytes + wq_attr.mp_rq.single_wqe_log_num_of_strides; + wqe_size = std::size_t(1) << log_wqe_size; + if (buffer_size < 2 * wqe_size) + buffer_size = 2 * wqe_size; + + bool reduced = false; + std::size_t strides = buffer_size >> wq_attr.mp_rq.single_stride_log_num_of_bytes; + if (std::size_t(device_attr.max_cqe) < strides) + { + strides = device_attr.max_cqe; + reduced = true; + } + std::size_t wqe = strides >> wq_attr.mp_rq.single_wqe_log_num_of_strides; + if (std::size_t(device_attr.max_qp_wr) < wqe) + { + wqe = device_attr.max_qp_wr; + reduced = true; + } + if (wqe < 2) + throw std::system_error(std::make_error_code(std::errc::not_supported), + "Insufficient resources for a multi-packet receive queue"); + buffer_size = wqe * wqe_size; + if (reduced) + log_warning("Reducing buffer to %1% to accommodate device limits", buffer_size); + this->buffer_size = buffer_size; + + ibv_exp_cq_init_attr cq_attr; + memset(&cq_attr, 0, sizeof(cq_attr)); + cq_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN; + cq_attr.res_domain = res_domain.get(); + if (comp_vector >= 0) + recv_cq = ibv_cq_t(cm_id, strides, nullptr, + comp_channel, comp_vector % cm_id->verbs->num_comp_vectors, + &cq_attr); + else + recv_cq = ibv_cq_t(cm_id, strides, nullptr, &cq_attr); + cq_intf = ibv_exp_cq_family_v1_t(cm_id, recv_cq); + + wq_attr.mp_rq.use_shift = IBV_EXP_MP_RQ_NO_SHIFT; + wq_attr.wq_type = IBV_EXP_WQT_RQ; + wq_attr.max_recv_wr = wqe; + wq_attr.max_recv_sge = 1; + wq_attr.pd = pd.get(); + wq_attr.cq = recv_cq.get(); + wq_attr.res_domain = res_domain.get(); + wq_attr.comp_mask = IBV_EXP_CREATE_WQ_MP_RQ | IBV_EXP_CREATE_WQ_RES_DOMAIN; + // TODO: investigate IBV_EXP_CREATE_WQ_FLAG_DELAY_DROP to reduce dropped + // packets. + wq = ibv_exp_wq_t(cm_id, &wq_attr); + wq_intf = ibv_exp_wq_family_t(cm_id, wq); + + rwq_ind_table = create_rwq_ind_table(cm_id, pd, wq); + qp = create_qp(cm_id, pd, recv_cq, rwq_ind_table, res_domain); + wq.modify(IBV_EXP_WQS_RDY); + + std::shared_ptr allocator = std::make_shared(0, true); + buffer = allocator->allocate(buffer_size, nullptr); + mr = ibv_mr_t(pd, buffer.get(), buffer_size, IBV_ACCESS_LOCAL_WRITE); + for (std::size_t i = 0; i < wqe; i++) + post_wr(i * wqe_size); + + flows = create_flows(qp, endpoints, cm_id->port_num); + enqueue_receive(true); + join_groups(endpoints, interface_address); +} + +udp_ibv_mprq_reader::udp_ibv_mprq_reader( + stream &owner, + const boost::asio::ip::udp::endpoint &endpoint, + const boost::asio::ip::address &interface_address, + std::size_t max_size, + std::size_t buffer_size, + int comp_vector, + int max_poll) + : udp_ibv_mprq_reader(owner, std::vector{endpoint}, + interface_address, max_size, buffer_size, comp_vector, max_poll) +{ +} + +} // namespace recv +} // namespace spead2 + +#endif // SPEAD2_USE_IBV diff -Nru spead2-1.10.0/src/recv_udp_pcap.cpp spead2-2.1.0/src/recv_udp_pcap.cpp --- spead2-1.10.0/src/recv_udp_pcap.cpp 2018-12-06 11:14:30.000000000 +0000 +++ spead2-2.1.0/src/recv_udp_pcap.cpp 2019-03-07 07:46:37.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016-2017 SKA South Africa +/* Copyright 2016-2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -38,11 +38,10 @@ { const int BATCH = 64; // maximum number of packets to process in one go - spead2::recv::stream_base &s = get_stream_base(); - spead2::recv::stream_base::add_packet_state state(s); + spead2::recv::stream_base::add_packet_state state(get_stream_base()); for (int pass = 0; pass < BATCH; pass++) { - if (s.is_stopped()) + if (state.is_stopped()) break; struct pcap_pkthdr *h; const u_char *pkt_data; @@ -78,13 +77,13 @@ break; case -2: // End of file - s.stop_received(); + state.stop(); break; } } // Run ourselves again - if (!s.is_stopped()) - get_stream().get_strand().post([this] { run(); }); + if (!state.is_stopped()) + get_io_service().post([this] { run(); }); else stopped(); } @@ -112,7 +111,7 @@ pcap_freecode(&filter); // Process the file - get_stream().get_strand().post([this] { run(); }); + get_io_service().post([this] { run(); }); } udp_pcap_file_reader::~udp_pcap_file_reader() diff -Nru spead2-1.10.0/src/send_inproc.cpp spead2-2.1.0/src/send_inproc.cpp --- spead2-1.10.0/src/send_inproc.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_inproc.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2018 SKA South Africa +/* Copyright 2018, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -44,11 +44,29 @@ } // namespace detail +void inproc_stream::async_send_packets() +{ + for (std::size_t i = 0; i < n_current_packets; i++) + { + inproc_queue::packet dup = detail::copy_packet(current_packets[i].pkt); + try + { + queue->buffer.push(std::move(dup)); + current_packets[i].result = boost::system::error_code(); + } + catch (ringbuffer_stopped &) + { + current_packets[i].result = boost::asio::error::operation_aborted; + } + } + get_io_service().post([this] { packets_handler(); }); +} + inproc_stream::inproc_stream( io_service_ref io_service, std::shared_ptr queue, const stream_config &config) - : stream_impl(std::move(io_service), config), + : stream_impl(std::move(io_service), config, 1), queue(std::move(queue)) { } diff -Nru spead2-1.10.0/src/send_packet.cpp spead2-2.1.0/src/send_packet.cpp --- spead2-1.10.0/src/send_packet.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_packet.cpp 2019-05-13 13:44:22.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -76,6 +76,8 @@ * to mark the separation between the padding and the last item. */ std::size_t item_packets = h.items.size() / max_item_pointers_per_packet + 1; + if (h.get_repeat_pointers() && item_packets > 1) + throw std::invalid_argument("packet size is too small to repeat item pointers"); /* We want every packet to have some payload, so that packets can be * unambiguously ordered and lost packets can be detected. For all * packets except the last, we want a multiple of sizeof(item_pointer_t) @@ -90,10 +92,21 @@ } } +bool packet_generator::has_next_packet() const +{ + return payload_offset < payload_size; +} + packet packet_generator::next_packet() { packet out; + if (h.get_repeat_pointers()) + { + next_item_pointer = 0; + next_address = 0; + } + if (payload_offset < payload_size) { pointer_encoder encoder(h.get_flavour().get_heap_address_bits()); diff -Nru spead2-1.10.0/src/send_streambuf.cpp spead2-2.1.0/src/send_streambuf.cpp --- spead2-1.10.0/src/send_streambuf.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_streambuf.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -26,11 +26,30 @@ namespace send { +void streambuf_stream::async_send_packets() +{ + for (std::size_t i = 0; i < n_current_packets; i++) + { + current_packets[i].result = boost::system::error_code(); + for (const auto &buffer : current_packets[i].pkt.buffers) + { + std::size_t buffer_size = boost::asio::buffer_size(buffer); + std::size_t written = streambuf.sputn(boost::asio::buffer_cast(buffer), buffer_size); + if (written != buffer_size) + { + current_packets[i].result = boost::asio::error::eof; + break; + } + } + } + get_io_service().post([this] { packets_handler(); }); +} + streambuf_stream::streambuf_stream( io_service_ref io_service, std::streambuf &streambuf, const stream_config &config) - : stream_impl(std::move(io_service), config), streambuf(streambuf) + : stream_impl(std::move(io_service), config, 64), streambuf(streambuf) { } diff -Nru spead2-1.10.0/src/send_stream.cpp spead2-2.1.0/src/send_stream.cpp --- spead2-1.10.0/src/send_stream.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_stream.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -38,11 +38,6 @@ this->max_packet_size = max_packet_size; } -std::size_t stream_config::get_max_packet_size() const -{ - return max_packet_size; -} - void stream_config::set_rate(double rate) { if (rate < 0.0 || !std::isfinite(rate)) @@ -50,11 +45,6 @@ this->rate = rate; } -double stream_config::get_rate() const -{ - return rate; -} - void stream_config::set_max_heaps(std::size_t max_heaps) { if (max_heaps == 0) @@ -62,21 +52,11 @@ this->max_heaps = max_heaps; } -std::size_t stream_config::get_max_heaps() const -{ - return max_heaps; -} - void stream_config::set_burst_size(std::size_t burst_size) { this->burst_size = burst_size; } -std::size_t stream_config::get_burst_size() const -{ - return burst_size; -} - void stream_config::set_burst_rate_ratio(double burst_rate_ratio) { if (burst_rate_ratio < 1.0 || !std::isfinite(burst_rate_ratio)) @@ -84,11 +64,6 @@ this->burst_rate_ratio = burst_rate_ratio; } -double stream_config::get_burst_rate_ratio() const -{ - return burst_rate_ratio; -} - double stream_config::get_burst_rate() const { return rate * burst_rate_ratio; @@ -118,5 +93,157 @@ { } + +std::size_t stream_impl_base::next_queue_slot(std::size_t cur) const +{ + if (++cur == config.get_max_heaps() + 1) + cur = 0; + return cur; +} + +stream_impl_base::queue_item *stream_impl_base::get_queue(std::size_t idx) +{ + return reinterpret_cast(queue.get() + idx); +} + +void stream_impl_base::next_active() +{ + active = next_queue_slot(active); + gen = boost::none; +} + +void stream_impl_base::post_handler(boost::system::error_code result) +{ + queue_item &front = *get_queue(queue_head); + get_io_service().post( + std::bind(std::move(front.handler), result, front.bytes_sent)); + if (active == queue_head) + { + // Can only happen if there is an error with the head of the queue + // before we've transmitted all its packets. + assert(result); + next_active(); + } + front.~queue_item(); + queue_head = next_queue_slot(queue_head); +} + +bool stream_impl_base::must_sleep() const +{ + return rate_bytes >= config.get_burst_size(); +} + +void stream_impl_base::process_results() +{ + for (std::size_t i = 0; i < n_current_packets; i++) + { + const transmit_packet &item = current_packets[i]; + if (item.item != get_queue(queue_head)) + { + // A previous packet in this heap already aborted it + continue; + } + if (item.result) + post_handler(item.result); + else + { + item.item->bytes_sent += item.size; + if (item.last) + post_handler(item.result); + } + } + n_current_packets = 0; +} + +stream_impl_base::timer_type::time_point stream_impl_base::update_send_times( + timer_type::time_point now) +{ + std::chrono::duration wait_burst(rate_bytes * seconds_per_byte_burst); + std::chrono::duration wait(rate_bytes * seconds_per_byte); + send_time_burst += std::chrono::duration_cast(wait_burst); + send_time += std::chrono::duration_cast(wait); + rate_bytes = 0; + + /* send_time_burst needs to reflect the time the burst + * was actually sent (as well as we can estimate it), even if + * send_time or now is later. + */ + timer_type::time_point target_time = std::max(send_time_burst, send_time); + send_time_burst = std::max(now, target_time); + return target_time; +} + +void stream_impl_base::update_send_time_empty() +{ + timer_type::time_point now = timer_type::clock_type::now(); + // Compute what send_time would need to be to make the next packet due to be + // transmitted now. + std::chrono::duration wait(rate_bytes * seconds_per_byte); + auto wait2 = std::chrono::duration_cast(wait); + timer_type::time_point backdate = now - wait2; + send_time = std::max(send_time, backdate); +} + +void stream_impl_base::load_packets(std::size_t tail) +{ + n_current_packets = 0; + while (n_current_packets < max_current_packets && !must_sleep() && active != tail) + { + queue_item *cur = get_queue(active); + if (!gen) + gen = boost::in_place(cur->h, cur->cnt, config.get_max_packet_size()); + assert(gen->has_next_packet()); + transmit_packet &data = current_packets[n_current_packets]; + data.pkt = gen->next_packet(); + data.size = boost::asio::buffer_size(data.pkt.buffers); + data.last = !gen->has_next_packet(); + data.item = cur; + data.result = boost::system::error_code(); + rate_bytes += data.size; + n_current_packets++; + if (data.last) + next_active(); + } +} + +stream_impl_base::stream_impl_base( + io_service_ref io_service, + const stream_config &config, + std::size_t max_current_packets) : + stream(std::move(io_service)), + current_packets(new transmit_packet[max_current_packets]), + max_current_packets(max_current_packets), + config(config), + seconds_per_byte_burst(config.get_burst_rate() > 0.0 ? 1.0 / config.get_burst_rate() : 0.0), + seconds_per_byte(config.get_rate() > 0.0 ? 1.0 / config.get_rate() : 0.0), + queue(new queue_item_storage[config.get_max_heaps() + 1]), + timer(get_io_service()) +{ +} + +stream_impl_base::~stream_impl_base() +{ + for (std::size_t i = queue_head; i != queue_tail; i = next_queue_slot(i)) + get_queue(i)->~queue_item(); +} + +void stream_impl_base::set_cnt_sequence(item_pointer_t next, item_pointer_t step) +{ + if (step == 0) + throw std::invalid_argument("step cannot be 0"); + std::unique_lock lock(queue_mutex); + next_cnt = next; + step_cnt = step; +} + +void stream_impl_base::flush() +{ + std::unique_lock lock(queue_mutex); + while (state != state_t::EMPTY) + { + heap_empty.wait(lock); + } +} + } // namespace send } // namespace spead2 diff -Nru spead2-1.10.0/src/send_tcp.cpp spead2-2.1.0/src/send_tcp.cpp --- spead2-1.10.0/src/send_tcp.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_tcp.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -27,6 +27,24 @@ namespace send { +void tcp_stream::async_send_packets() +{ + if (!connected.load()) + { + current_packets[0].result = boost::asio::error::not_connected; + get_io_service().post([this] { packets_handler(); }); + } + else + { + auto handler = [this](const boost::system::error_code &ec, std::size_t) + { + current_packets[0].result = ec; + packets_handler(); + }; + boost::asio::async_write(socket, current_packets[0].pkt.buffers, handler); + } +} + namespace detail { @@ -51,20 +69,22 @@ io_service_ref io_service, boost::asio::ip::tcp::socket &&socket, const stream_config &config) - : stream_impl(std::move(io_service), config), + : stream_impl(std::move(io_service), config, 1), socket(std::move(socket)), connected(true) { - if (&get_io_service() != &this->socket.get_io_service()) + if (!socket_uses_io_service(this->socket, get_io_service())) throw std::invalid_argument("I/O service does not match the socket's I/O service"); } +#if BOOST_VERSION < 107000 tcp_stream::tcp_stream( boost::asio::ip::tcp::socket &&socket, const stream_config &config) - : tcp_stream(socket.get_io_service(), std::move(socket), config) + : tcp_stream(get_socket_io_service(socket), std::move(socket), config) { } +#endif tcp_stream::~tcp_stream() { diff -Nru spead2-1.10.0/src/send_udp.cpp spead2-2.1.0/src/send_udp.cpp --- spead2-1.10.0/src/send_udp.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_udp.cpp 2019-11-13 08:46:10.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -28,6 +29,83 @@ constexpr std::size_t udp_stream::default_buffer_size; +void udp_stream::send_packets(std::size_t first) +{ +#if SPEAD2_USE_SENDMMSG + // Try synchronous send + if (first < n_current_packets) + { + int sent = sendmmsg(socket.native_handle(), msgvec + first, n_current_packets - first, MSG_DONTWAIT); + if (sent < 0 && errno != EAGAIN && errno != EWOULDBLOCK) + { + current_packets[first].result = boost::system::error_code(errno, boost::asio::error::get_system_category()); + first++; + } + else if (sent > 0) + { + for (int i = 0; i < sent; i++) + current_packets[first + i].result = boost::system::error_code(); + first += sent; + } + if (first < n_current_packets) + { + socket.async_send(boost::asio::null_buffers(), [this, first](const boost::system::error_code &ec, std::size_t) + { + send_packets(first); + }); + return; + } + } +#else + for (std::size_t idx = first; idx < n_current_packets; idx++) + { + // First try to send synchronously, to reduce overheads from callbacks etc + boost::system::error_code ec; + socket.send_to(current_packets[idx].pkt.buffers, endpoint, 0, ec); + if (ec == boost::asio::error::would_block) + { + // Socket buffer is full, fall back to asynchronous + auto handler = [this, idx](const boost::system::error_code &ec, std::size_t bytes_transferred) + { + current_packets[idx].result = ec; + send_packets(idx + 1); + }; + socket.async_send_to(current_packets[idx].pkt.buffers, endpoint, handler); + return; + } + else + { + current_packets[idx].result = ec; + } + } +#endif + + get_io_service().post([this] { packets_handler(); }); +} + +void udp_stream::async_send_packets() +{ +#if SPEAD2_USE_SENDMMSG + msg_iov.clear(); + for (std::size_t i = 0; i < n_current_packets; i++) + for (const auto &buffer : current_packets[i].pkt.buffers) + { + msg_iov.push_back(iovec{const_cast(boost::asio::buffer_cast(buffer)), + boost::asio::buffer_size(buffer)}); + } + // Assigning msgvec must be done in a second pass, because appending to + // msg_iov invalidates references. + std::size_t offset = 0; + for (std::size_t i = 0; i < n_current_packets; i++) + { + msgvec[i].msg_hdr.msg_iov = &msg_iov[offset]; + msgvec[i].msg_hdr.msg_iovlen = current_packets[i].pkt.buffers.size(); + offset += msgvec[i].msg_hdr.msg_iovlen; + } +#endif + send_packets(0); +} + static boost::asio::ip::udp::socket make_socket( boost::asio::io_service &io_service, const boost::asio::ip::udp &protocol, @@ -131,6 +209,7 @@ { } +#if BOOST_VERSION < 107000 udp_stream::udp_stream( boost::asio::ip::udp::socket &&socket, const boost::asio::ip::udp::endpoint &endpoint, @@ -147,6 +226,7 @@ : udp_stream(socket.get_io_service(), std::move(socket), endpoint, config) { } +#endif udp_stream::udp_stream( io_service_ref io_service, @@ -154,12 +234,22 @@ const boost::asio::ip::udp::endpoint &endpoint, const stream_config &config, std::size_t buffer_size) - : stream_impl(std::move(io_service), config), + : stream_impl(std::move(io_service), config, batch_size), socket(std::move(socket)), endpoint(endpoint) { - if (&get_io_service() != &this->socket.get_io_service()) + if (!socket_uses_io_service(this->socket, get_io_service())) throw std::invalid_argument("I/O service does not match the socket's I/O service"); set_socket_send_buffer_size(this->socket, buffer_size); + this->socket.non_blocking(true); +#if SPEAD2_USE_SENDMMSG + std::memset(&msgvec, 0, sizeof(msgvec)); + for (std::size_t i = 0; i < batch_size; i++) + { + auto &hdr = msgvec[i].msg_hdr; + hdr.msg_name = (void *) this->endpoint.data(); + hdr.msg_namelen = this->endpoint.size(); + } +#endif // SPEAD2_USE_SENDMMSG } udp_stream::udp_stream( diff -Nru spead2-1.10.0/src/send_udp_ibv.cpp spead2-2.1.0/src/send_udp_ibv.cpp --- spead2-1.10.0/src/send_udp_ibv.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/send_udp_ibv.cpp 2020-02-14 08:26:39.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016 SKA South Africa +/* Copyright 2016, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -55,104 +55,124 @@ void udp_ibv_stream::reap() { - ibv_wc wc; + constexpr int BATCH = 16; + ibv_wc wc[BATCH]; int done; - while ((done = send_cq.poll(1, &wc)) > 0) + while ((done = send_cq.poll(BATCH, wc)) > 0) { - if (wc.status != IBV_WC_SUCCESS) + for (int i = 0; i < done; i++) { - log_warning("Work Request failed with code %1%", wc.status); + if (wc[i].status != IBV_WC_SUCCESS) + { + log_warning("Work Request failed with code %1%", wc[i].status); + } + slot *s = &slots[wc[i].wr_id]; + available.push_back(s); } - slot *s = &slots[wc.wr_id]; - available.push_back(s); } } -udp_ibv_stream::rerun_async_send_packet::rerun_async_send_packet( - udp_ibv_stream *self, - const packet &pkt, - udp_ibv_stream::completion_handler &&handler) - : self(self), pkt(&pkt), handler(std::move(handler)) +bool udp_ibv_stream::make_space() { -} + for (int i = 0; i < max_poll; i++) + { + reap(); + if (available.size() >= n_current_packets) + return true; + } -void udp_ibv_stream::rerun_async_send_packet::operator()( - boost::system::error_code ec, std::size_t bytes_transferred) -{ - (void) bytes_transferred; - if (ec) + // Synchronous attempts failed. Give the event loop a chance to run. + if (comp_channel) { - handler(ec, 0); + send_cq.req_notify(false); + + auto rerun = [this] (const boost::system::error_code &ec, size_t) + { + if (ec) + { + for (std::size_t i = 0; i < n_current_packets; i++) + current_packets[i].result = ec; + packets_handler(); + } + else + { + ibv_cq *event_cq; + void *event_cq_context; + // This should be non-blocking, since we were woken up, but + // spurious wakeups have been observed. + while (comp_channel.get_event(&event_cq, &event_cq_context)) + send_cq.ack_events(1); + async_send_packets(); + } + }; + + /* Need to check again, in case of a race (in which case the event + * won't fire until we send some more packets). Note that this + * leaves us with an unwanted req_notify which will lead to a + * spurious event later, but that is harmless. + */ + reap(); + if (available.size() >= n_current_packets) + return true; + comp_channel_wrapper.async_read_some(boost::asio::null_buffers(), rerun); } else { - ibv_cq *event_cq; - void *event_cq_context; - // This should be non-blocking, since we were woken up - self->comp_channel.get_event(&event_cq, &event_cq_context); - self->send_cq.ack_events(1); - self->async_send_packet(*pkt, std::move(handler)); + get_io_service().post([this] { async_send_packets(); }); } -} - -udp_ibv_stream::invoke_handler::invoke_handler( - udp_ibv_stream::completion_handler &&handler, - boost::system::error_code ec, - std::size_t bytes_transferred) - : handler(std::move(handler)), ec(ec), bytes_transferred(bytes_transferred) -{ -} -void udp_ibv_stream::invoke_handler::operator()() -{ - handler(ec, bytes_transferred); + return false; } -void udp_ibv_stream::async_send_packet(const packet &pkt, completion_handler &&handler) +void udp_ibv_stream::async_send_packets() { try { - reap(); - if (available.empty()) + if (!make_space()) + return; + + slot *prev = nullptr; + slot *first = nullptr; + for (std::size_t i = 0; i < n_current_packets; i++) { - if (comp_channel) - { - send_cq.req_notify(false); - // Need to check again, in case of a race - reap(); - comp_channel_wrapper.async_read_some( - boost::asio::null_buffers(), - rerun_async_send_packet(this, pkt, std::move(handler))); - return; - } + const auto ¤t_packet = current_packets[i]; + slot *s = available.back(); + available.pop_back(); + + std::size_t payload_size = current_packet.size; + ipv4_packet ipv4 = s->frame.payload_ipv4(); + ipv4.total_length(payload_size + udp_packet::min_size + ipv4.header_length()); + ipv4.update_checksum(); + udp_packet udp = ipv4.payload_udp(); + udp.length(payload_size + udp_packet::min_size); + packet_buffer payload = udp.payload(); + boost::asio::buffer_copy(boost::asio::mutable_buffer(payload), current_packet.pkt.buffers); + s->sge.length = payload_size + (payload.data() - s->frame.data()); + s->wr.next = nullptr; + if (prev != nullptr) + prev->wr.next = &s->wr; else - { - // Poll mode - keep trying until we have space - while (available.empty()) - reap(); - } + first = s; + prev = s; } - slot *s = available.back(); - available.pop_back(); - - std::size_t payload_size = boost::asio::buffer_size(pkt.buffers); - ipv4_packet ipv4 = s->frame.payload_ipv4(); - ipv4.total_length(payload_size + udp_packet::min_size + ipv4.header_length()); - ipv4.update_checksum(); - udp_packet udp = ipv4.payload_udp(); - udp.length(payload_size + udp_packet::min_size); - packet_buffer payload = udp.payload(); - boost::asio::buffer_copy(boost::asio::mutable_buffer(payload), pkt.buffers); - s->sge.length = payload_size + (payload.data() - s->frame.data()); - qp.post_send(&s->wr); - get_io_service().post(invoke_handler(std::move(handler), boost::system::error_code(), - payload_size)); + qp.post_send(&first->wr); + // TODO: wait until we've reaped the CQE to claim success and post + // completion? + for (std::size_t i = 0; i < n_current_packets; i++) + current_packets[i].result = boost::system::error_code(); } catch (std::system_error &e) { - get_io_service().post(invoke_handler(std::move(handler), - boost::system::error_code(e.code().value(), boost::system::system_category()), 0)); + boost::system::error_code ec(e.code().value(), boost::system::system_category()); + for (std::size_t i = 0; i < n_current_packets; i++) + current_packets[i].result = ec; } + get_io_service().post([this] { packets_handler(); }); +} + +static std::size_t calc_n_slots(const stream_config &config, std::size_t buffer_size) +{ + return std::max(std::size_t(1), buffer_size / (config.get_max_packet_size() + header_length)); } udp_ibv_stream::udp_ibv_stream( @@ -164,11 +184,13 @@ int ttl, int comp_vector, int max_poll) - : stream_impl(std::move(io_service), config), - n_slots(std::max(std::size_t(1), buffer_size / (config.get_max_packet_size() + header_length))), + : stream_impl(std::move(io_service), config, + std::max(std::size_t(1), calc_n_slots(config, buffer_size) / 2)), + n_slots(calc_n_slots(config, buffer_size)), socket(get_io_service(), endpoint.protocol()), cm_id(event_channel, nullptr, RDMA_PS_UDP), - comp_channel_wrapper(get_io_service()) + comp_channel_wrapper(get_io_service()), + max_poll(max_poll) { if (!endpoint.address().is_v4() || !endpoint.address().is_multicast()) throw std::invalid_argument("endpoint is not an IPv4 multicast address"); diff -Nru spead2-1.10.0/src/spead2_bench.cpp spead2-2.1.0/src/spead2_bench.cpp --- spead2-1.10.0/src/spead2_bench.cpp 2018-12-04 09:49:56.000000000 +0000 +++ spead2-2.1.0/src/spead2_bench.cpp 2020-02-14 08:26:39.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015, 2017 SKA South Africa +/* Copyright 2015, 2017, 2019-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -237,47 +238,70 @@ return out; } -static std::int64_t send_heaps(spead2::send::stream &stream, - const std::vector &heaps) +namespace +{ + +class sender +{ +private: + spead2::send::stream &stream; + const std::vector &heaps; + const std::size_t max_heaps; + + std::uint64_t bytes_transferred = 0; + boost::system::error_code error; + spead2::semaphore done_sem{0}; + + void callback(std::size_t idx, const boost::system::error_code &ec, std::size_t bytes_transferred); + +public: + sender(spead2::send::stream &stream, const std::vector &heaps, + const options &opts); + std::int64_t run(); +}; + +sender::sender(spead2::send::stream &stream, const std::vector &heaps, + const options &opts) + : stream(stream), heaps(heaps), max_heaps(std::min(opts.heaps, heaps.size())) { - std::size_t n_heaps = heaps.size(); - std::deque> futures; - std::deque> promises; - std::int64_t transferred = 0; - boost::system::error_code last_error; - for (std::size_t i = 0; i < n_heaps; i++) - { - while (futures.size() >= 2) - { - transferred += futures.front().get(); - futures.pop_front(); - promises.pop_front(); - } - promises.emplace_back(); - std::promise &promise = promises.back(); - futures.push_back(promise.get_future()); - auto callback = [&transferred, &last_error, &promise] ( - const boost::system::error_code &ec, spead2::item_pointer_t bytes) - { - if (!ec) - promise.set_value(bytes); - else - last_error = ec; - }; - stream.async_send_heap(heaps[i], callback); - } - stream.flush(); - while (!futures.empty()) - { - transferred += futures.front().get(); - futures.pop_front(); - promises.pop_front(); - } - if (last_error) - throw boost::system::system_error(last_error); - return transferred; } +void sender::callback(std::size_t idx, const boost::system::error_code &ec, std::size_t bytes_transferred) +{ + this->bytes_transferred += bytes_transferred; + if (ec && !error) + error = ec; + if (!error && idx + max_heaps < heaps.size()) + { + idx += max_heaps; + stream.async_send_heap(heaps[idx], [this, idx] (const boost::system::error_code &ec, std::size_t bytes_transferred) { + callback(idx, ec, bytes_transferred); }); + } + else + done_sem.put(); +} + +std::int64_t sender::run() +{ + bytes_transferred = 0; + error = boost::system::error_code(); + /* See comments in spead2_send.cpp for the explanation of why this is + * posted rather than run directly. + */ + stream.get_io_service().post([this] { + for (int i = 0; i < max_heaps; i++) + stream.async_send_heap(heaps[i], [this, i] (const boost::system::error_code &ec, std::size_t bytes_transferred) { + callback(i, ec, bytes_transferred); }); + }); + for (int i = 0; i < max_heaps; i++) + semaphore_get(done_sem); + if (error) + throw boost::system::system_error(error); + return bytes_transferred; +} + +} // anonymous namespace + static std::pair measure_connection_once( const options &opts, double rate, std::int64_t num_heaps, std::int64_t required_heaps) @@ -325,11 +349,8 @@ /* Construct the stream */ spead2::thread_pool thread_pool; spead2::flavour flavour(4, 64, opts.heap_address_bits); - /* Allow all heaps to be queued up at once. Since they all point to - * the same payload, this should not cause excessive memory use. - */ spead2::send::stream_config config( - opts.packet_size, rate, opts.burst_size, num_heaps + 1, opts.burst_rate_ratio); + opts.packet_size, rate, opts.burst_size, opts.heaps, opts.burst_rate_ratio); /* Build the heaps */ std::vector heaps; @@ -360,7 +381,8 @@ stream.reset(new spead2::send::udp_stream( thread_pool.get_io_service(), endpoint, config, opts.send_buffer)); } - transferred = send_heaps(*stream, heaps); + sender s(*stream, heaps, opts); + transferred = s.run(); auto end = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed_duration = end - start; double actual_rate = transferred / elapsed_duration.count(); @@ -470,11 +492,8 @@ virtual void stop_received() override { - if (!is_stopped()) - { - spead2::recv::stream::stop_received(); - stopped_promise.set_value(); - } + spead2::recv::stream::stop_received(); + stopped_promise.set_value(); } public: @@ -668,8 +687,10 @@ opts.recv_ibv_if = decode_string(opts.recv_ibv_if); /* Look up the bind address for the data socket */ udp::resolver resolver(thread_pool.get_io_service()); - udp::resolver::query query(opts.multicast.empty() ? "0.0.0.0" : opts.multicast, - slave_opts.port); + udp::resolver::query query( + !opts.multicast.empty() ? opts.multicast : + !opts.recv_ibv_if.empty() ? opts.recv_ibv_if : "0.0.0.0", + slave_opts.port); udp::endpoint endpoint = *resolver.resolve(query); if (opts.ring) connection.reset(new recv_connection_ring(opts)); diff -Nru spead2-1.10.0/src/spead2_recv.cpp spead2-2.1.0/src/spead2_recv.cpp --- spead2-1.10.0/src/spead2_recv.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/spead2_recv.cpp 2020-02-12 12:27:05.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2015 SKA South Africa +/* Copyright 2015, 2018-2020 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -28,9 +28,6 @@ #include #include #include -#if SPEAD2_USE_NETMAP -# include -#endif #if SPEAD2_USE_IBV # include #endif @@ -64,9 +61,6 @@ std::size_t mem_initial = 8; bool ring = false; bool memcpy_nt = false; -#if SPEAD2_USE_NETMAP - bool netmap = false; -#endif #if SPEAD2_USE_IBV bool ibv = false; int ibv_comp_vector = 0; @@ -112,7 +106,7 @@ ("pyspead", make_opt(opts.pyspead), "Be bug-compatible with PySPEAD") ("joint", make_opt(opts.joint), "Treat all sources as a single stream") ("tcp", make_opt(opts.tcp), "Receive data over TCP instead of UDP") - ("bind", make_opt(opts.bind), "Interface address for multicast") + ("bind", make_opt(opts.bind), "Interface address") ("packet", make_opt_no_default(opts.packet), "Maximum packet size to use") ("buffer", make_opt_no_default(opts.buffer), "Socket buffer size") ("threads", make_opt(opts.threads), "Number of worker threads") @@ -125,9 +119,6 @@ ("mem-initial", make_opt(opts.mem_initial), "Initial free memory buffers") ("ring", make_opt(opts.ring), "Use ringbuffer instead of callbacks") ("memcpy-nt", make_opt(opts.memcpy_nt), "Use non-temporal memcpy") -#if SPEAD2_USE_NETMAP - ("netmap", make_opt(opts.netmap), "Use netmap") -#endif #if SPEAD2_USE_IBV ("ibv", make_opt(opts.ibv), "Use ibverbs") ("ibv-vector", make_opt(opts.ibv_comp_vector), "Interrupt vector (-1 for polled)") @@ -179,16 +170,6 @@ if (opts.tcp && opts.ibv) throw po::error("--ibv and --tcp are incompatible"); #endif -#if SPEAD2_USE_NETMAP - if (opts.sources.size() > 1 && opts.netmap) - throw po::error("--netmap cannot be used with multiple sources"); - if (opts.tcp && opts.netmap) - throw po::error("--netmap and --tcp are incompatible"); -#endif -#if SPEAD2_USE_IBV && SPEAD2_USE_NETMAP - if (opts.ibv && opts.netmap) - throw po::error("--ibv and --netmap are incompatible"); -#endif return opts; } catch (po::error &e) @@ -247,7 +228,10 @@ for (const auto &item : items) { std::cout << std::hex << item.id << std::dec - << " = [" << item.length << " bytes]\n"; + << " = [" << item.length << " bytes]"; + if (item.is_immediate) + std::cout << " = " << std::hex << item.immediate_value; + std::cout << '\n'; } std::cout << std::noshowbase; } @@ -278,6 +262,11 @@ : spead2::recv::stream::stream(std::forward(args)...), opts(opts) {} + ~callback_stream() + { + stop(); + } + virtual void stop_received() override { spead2::recv::stream::stop_received(); @@ -336,7 +325,7 @@ { boost::lexical_cast(port); } - catch (boost::bad_lexical_cast) + catch (boost::bad_lexical_cast &) { is_pcap = true; } @@ -359,16 +348,10 @@ else { udp::resolver resolver(thread_pool.get_io_service()); - udp::resolver::query query(host, port); + udp::resolver::query query(host, port, + boost::asio::ip::udp::resolver::query::address_configured + | boost::asio::ip::udp::resolver::query::passive); udp::endpoint endpoint = *resolver.resolve(query); -#if SPEAD2_USE_NETMAP - if (opts.netmap) - { - stream->emplace_reader( - opts.bind, endpoint.port()); - } - else -#endif #if SPEAD2_USE_IBV if (opts.ibv) { @@ -376,8 +359,7 @@ } else #endif - if (endpoint.address().is_multicast() && endpoint.address().is_v4() - && !opts.bind.empty()) + if (endpoint.address().is_v4() && !opts.bind.empty()) { stream->emplace_reader( endpoint, opts.packet, opts.buffer, @@ -386,7 +368,7 @@ else { if (!opts.bind.empty()) - std::cerr << "--bind is only applicable to IPv4 multicast, ignoring\n"; + std::cerr << "--bind is not implemented for IPv6\n"; stream->emplace_reader(endpoint, opts.packet, opts.buffer); } } @@ -424,6 +406,17 @@ streams.push_back(make_stream(thread_pool, opts, it, it + 1)); } + spead2::thread_pool stopper_thread_pool; + boost::asio::signal_set signals(stopper_thread_pool.get_io_service()); + signals.add(SIGINT); + signals.async_wait([&streams] (const boost::system::error_code &error, int signal_number) { + if (!error) + for (const std::unique_ptr &stream : streams) + { + stream->stop(); + } + }); + std::int64_t n_complete = 0; if (opts.ring) { @@ -450,9 +443,17 @@ n_complete += stream.join(); } } + signals.cancel(); spead2::recv::stream_stats stats; - for (const auto &ptr : streams) + for (auto &ptr : streams) + { + /* Even though we've seen the stop condition, if we don't explicitly + * stop the stream then a race condition means we might not see the + * last batch of statistics updates. + */ + ptr->stop(); stats += ptr->get_stats(); + } std::cout << "Received " << n_complete << " heaps\n"; #define REPORT_STAT(field) (std::cout << #field ": " << stats.field << '\n') diff -Nru spead2-1.10.0/src/spead2_send.cpp spead2-2.1.0/src/spead2_send.cpp --- spead2-1.10.0/src/spead2_send.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/spead2_send.cpp 2020-02-14 08:26:39.000000000 +0000 @@ -1,4 +1,4 @@ -/* Copyright 2016, 2017 SKA South Africa +/* Copyright 2016, 2017, 2019 SKA South Africa * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,7 @@ std::size_t buffer; std::size_t burst = spead2::send::stream_config::default_burst_size; double burst_rate_ratio = spead2::send::stream_config::default_burst_rate_ratio; - int threads = 1; + std::size_t max_heaps = spead2::send::stream_config::default_max_heaps; double rate = 0.0; int ttl = 1; #if SPEAD2_USE_IBV @@ -104,7 +105,7 @@ ("buffer", make_opt_no_default(opts.buffer), "Socket buffer size") ("burst", make_opt(opts.burst), "Burst size") ("burst-rate-ratio", make_opt(opts.burst_rate_ratio), "Hard rate limit, relative to --rate") - ("threads", make_opt(opts.threads), "Number of worker threads") + ("max-heaps", make_opt(opts.max_heaps), "Maximum heaps in flight") ("rate", make_opt(opts.rate), "Transmission rate bound (Gb/s)") ("ttl", make_opt(opts.ttl), "TTL for multicast target") #if SPEAD2_USE_IBV @@ -160,37 +161,57 @@ } } -// Sends a heap, returning a future instead of using a completion handler -std::future async_send_heap(spead2::send::stream &stream, const spead2::send::heap &heap) +namespace { - auto promise = std::make_shared>(); - auto handler = [promise] (boost::system::error_code ec, std::size_t bytes_transferred) - { - if (ec) - promise->set_exception(std::make_exception_ptr(boost::system::system_error(ec))); - else - promise->set_value(bytes_transferred); - }; - stream.async_send_heap(heap, handler); - return promise->get_future(); -} -int run(spead2::send::stream &stream, const options &opts) +class sender { +private: + spead2::send::stream &stream; + const std::size_t max_heaps; + const std::int64_t n_heaps; + const spead2::flavour flavour; + + spead2::send::heap first_heap; // has descriptors + std::vector heaps; + spead2::send::heap last_heap; // has end-of-stream marker typedef std::pair item_t; - std::size_t elements = opts.heap_size / (opts.items * sizeof(item_t)); - std::size_t heap_size = elements * opts.items * sizeof(item_t); + std::vector> values; + + std::uint64_t bytes_transferred = 0; + boost::system::error_code error; + spead2::semaphore done_sem{0}; + + const spead2::send::heap &get_heap(std::uint64_t idx) const noexcept; + + void callback(std::uint64_t idx, const boost::system::error_code &ec, std::size_t bytes_transferred); + +public: + sender(spead2::send::stream &stream, const options &opts); + std::uint64_t run(); +}; + +sender::sender(spead2::send::stream &stream, const options &opts) + : stream(stream), + max_heaps((opts.heaps < 0 || std::uint64_t(opts.heaps) >= opts.max_heaps) + ? opts.max_heaps : opts.heaps + 1), + n_heaps(opts.heaps), + flavour(spead2::maximum_version, 64, opts.addr_bits, + opts.pyspead ? spead2::BUG_COMPAT_PYSPEAD_0_5_2 : 0), + first_heap(flavour), + last_heap(flavour) +{ + heaps.reserve(max_heaps); + for (std::size_t i = 0; i < max_heaps; i++) + heaps.emplace_back(flavour); + + const std::size_t elements = opts.heap_size / (opts.items * sizeof(item_t)); + const std::size_t heap_size = elements * opts.items * sizeof(item_t); if (heap_size != opts.heap_size) { std::cerr << "Heap size is not an exact multiple: using " << heap_size << " instead of " << opts.heap_size << '\n'; } - spead2::flavour f(spead2::maximum_version, 64, opts.addr_bits, - opts.pyspead ? spead2::BUG_COMPAT_PYSPEAD_0_5_2 : 0); - - std::vector descriptors; - std::vector> values; - descriptors.reserve(opts.items); values.reserve(opts.items); for (std::size_t i = 0; i < opts.items; i++) { @@ -203,50 +224,78 @@ sstr.str(""); sstr << "{'shape': (" << elements << ",), 'fortran_order': False, 'descr': ' item(new item_t[elements]); + item_t *ptr = item.get(); values.push_back(std::move(item)); + for (std::size_t j = 0; j < max_heaps; j++) + heaps[j].add_item(0x1000 + i, ptr, elements * sizeof(item_t), true); + first_heap.add_item(0x1000 + i, ptr, elements * sizeof(item_t), true); } + last_heap.add_end(); +} - std::deque> futures; - std::deque heaps; - std::uint64_t sent = 0; - std::uint64_t sent_bytes = 0; - auto start_time = std::chrono::high_resolution_clock::now(); - while (opts.heaps < 0 || sent <= std::uint64_t(opts.heaps)) - { - if (futures.size() >= 2) - { - sent_bytes += futures.front().get(); - futures.pop_front(); - heaps.pop_front(); - } +const spead2::send::heap &sender::get_heap(std::uint64_t idx) const noexcept +{ + if (idx == 0) + return first_heap; + else if (n_heaps >= 0 && idx == std::uint64_t(n_heaps)) + return last_heap; + else + return heaps[idx % max_heaps]; +} - heaps.emplace_back(f); - spead2::send::heap &heap = heaps.back(); - if (sent == 0) - { - for (const auto &d : descriptors) - heap.add_descriptor(d); - } - if (opts.heaps >= 0 && sent == std::uint64_t(opts.heaps)) - { - heap.add_end(); - } - else - { - for (std::size_t i = 0; i < opts.items; i++) - heap.add_item(0x1000 + i, values[i].get(), elements * sizeof(item_t), true); - } - futures.push_back(async_send_heap(stream, heap)); - sent++; +void sender::callback(std::uint64_t idx, const boost::system::error_code &ec, std::size_t bytes_transferred) +{ + this->bytes_transferred += bytes_transferred; + if (ec && !error) + error = ec; + if (error) + { + done_sem.put(); + return; } - while (futures.size() > 0) + + if (n_heaps == -1 || std::uint64_t(n_heaps) - idx >= max_heaps) { - sent_bytes += futures.front().get(); - futures.pop_front(); - heaps.pop_front(); + idx += max_heaps; + stream.async_send_heap(get_heap(idx), [this, idx] (const boost::system::error_code &ec, std::size_t bytes_transferred) { + callback(idx, ec, bytes_transferred); }); } + else + done_sem.put(); +} + +std::uint64_t sender::run() +{ + bytes_transferred = 0; + error = boost::system::error_code(); + /* Send the initial heaps from the worker thread. This ensures that no + * callbacks can happen until the initial heaps are all sent, which would + * otherwise lead to heaps being queued out of order. For this benchmark it + * doesn't really matter since the heaps are all the same, but it makes it + * a more realistic benchmark. + */ + stream.get_io_service().post([this] { + for (int i = 0; i < max_heaps; i++) + stream.async_send_heap(get_heap(i), [this, i] (const boost::system::error_code &ec, std::size_t bytes_transferred) { + callback(i, ec, bytes_transferred); }); + }); + for (int i = 0; i < max_heaps; i++) + semaphore_get(done_sem); + if (error) + throw boost::system::system_error(error); + return bytes_transferred; +} + +} // anonymous namespace + +static int run(spead2::send::stream &stream, const options &opts) +{ + sender s(stream, opts); + + auto start_time = std::chrono::high_resolution_clock::now(); + std::uint64_t sent_bytes = s.run(); auto stop_time = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = stop_time - start_time; double elapsed_s = elapsed.count(); @@ -257,7 +306,8 @@ } template -boost::asio::ip::basic_endpoint get_endpoint(boost::asio::io_service &io_service, const options &opts) +static boost::asio::ip::basic_endpoint get_endpoint( + boost::asio::io_service &io_service, const options &opts) { typedef boost::asio::ip::basic_resolver resolver_type; resolver_type resolver(io_service); @@ -269,10 +319,10 @@ { options opts = parse_args(argc, argv); - spead2::thread_pool thread_pool(opts.threads); + spead2::thread_pool thread_pool(1); spead2::send::stream_config config( opts.packet, opts.rate * 1000 * 1000 * 1000 / 8, opts.burst, - spead2::send::stream_config::default_max_heaps, opts.burst_rate_ratio); + opts.max_heaps, opts.burst_rate_ratio); std::unique_ptr stream; auto &io_service = thread_pool.get_io_service(); boost::asio::ip::address interface_address; @@ -282,7 +332,7 @@ if (opts.tcp) { tcp::endpoint endpoint = get_endpoint(io_service, opts); auto promise = std::promise(); - auto connect_handler = [&promise] (boost::system::error_code e) { + auto connect_handler = [&promise] (const boost::system::error_code &e) { if (e) promise.set_exception(std::make_exception_ptr(boost::system::system_error(e))); else diff -Nru spead2-1.10.0/src/unittest_logging.cpp spead2-2.1.0/src/unittest_logging.cpp --- spead2-1.10.0/src/unittest_logging.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/unittest_logging.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -0,0 +1,181 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for common_logging. + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ +namespace unittest +{ + +struct capture_logging +{ + std::function orig_log_function; + std::vector levels; + std::vector messages; + + capture_logging() + : orig_log_function(set_log_function( + [this] (log_level level, const std::string &msg) + { + levels.push_back(level); + messages.push_back(msg); + })) + { + } + + ~capture_logging() + { + set_log_function(orig_log_function); + } +}; + +struct capture_stderr +{ + std::ostringstream out; + std::streambuf *old_buf; + + capture_stderr() + { + old_buf = std::cerr.rdbuf(out.rdbuf()); + } + + ~capture_stderr() + { + std::cerr.rdbuf(old_buf); + } +}; + +BOOST_AUTO_TEST_SUITE(common) +BOOST_FIXTURE_TEST_SUITE(logging, capture_logging) + +#define CHECK_MESSAGES(message, level) do { \ + std::vector expected_messages{(message)}; \ + BOOST_CHECK_EQUAL_COLLECTIONS(messages.begin(), messages.end(), \ + expected_messages.begin(), expected_messages.end()); \ + std::vector expected_levels{(level)}; \ + BOOST_CHECK_EQUAL_COLLECTIONS(levels.begin(), levels.end(), \ + expected_levels.begin(), expected_levels.end()); \ + } while (false) + +BOOST_AUTO_TEST_CASE(log_info) +{ + spead2::log_info("Hello %1%", 3); + CHECK_MESSAGES("Hello 3", log_level::info); +} + +BOOST_AUTO_TEST_CASE(log_errno_explicit) +{ + errno = 0; + log_errno("Test: %1% %2%", EBADF); + std::ostringstream expected; + expected << "Test: " << EBADF << " Bad file descriptor"; + CHECK_MESSAGES(expected.str(), log_level::warning); +} + +BOOST_AUTO_TEST_CASE(log_errno_implicit) +{ + errno = EBADF; + spead2::log_errno("Test: %1% %2%"); + std::ostringstream expected; + expected << "Test: " << EBADF << " Bad file descriptor"; + CHECK_MESSAGES(expected.str(), log_level::warning); +} + +static boost::test_tools::predicate_result ebadf_exception(const std::system_error &error) +{ + boost::test_tools::predicate_result result(false); + if (error.code().value() != EBADF) + { + result.message() << "Expected error code " << EBADF << ", got " << error.code().value(); + return result; + } + if (error.code().category() != std::system_category()) + { + result.message() << "Incorrect error category"; + return result; + } + std::string what = error.what(); + if (what.find("blah") == std::string::npos) + { + result.message() << "Did not find 'blah' in '" << what << "'"; + return result; + } + return true; +} + +static boost::test_tools::predicate_result no_error_exception(const std::system_error &error) +{ + boost::test_tools::predicate_result result(false); + if (error.code().default_error_condition() != std::errc::invalid_argument) + { + result.message() << "Expected error condition invalid_argument, got " + << error.code().value(); + return result; + } + std::string what = error.what(); + if (what.find("blah") == std::string::npos) + { + result.message() << "Did not find 'blah' in '" << what << "'"; + return result; + } + if (what.find("(unknown error)") == std::string::npos) + { + result.message() << "Did not find '(unknown error)' in '" << what << "'"; + return result; + } + return true; +} + +BOOST_AUTO_TEST_CASE(throw_errno_explicit) +{ + errno = 0; + BOOST_CHECK_EXCEPTION(throw_errno("blah", EBADF), std::system_error, ebadf_exception); +} + +BOOST_AUTO_TEST_CASE(throw_errno_implicit) +{ + errno = EBADF; + BOOST_CHECK_EXCEPTION(throw_errno("blah"), std::system_error, ebadf_exception); +} + +BOOST_AUTO_TEST_CASE(throw_errno_zero) +{ + BOOST_CHECK_EXCEPTION(throw_errno("blah", 0), std::system_error, no_error_exception); +} + +BOOST_FIXTURE_TEST_CASE(default_logger, capture_stderr) +{ + spead2::log_info("A test message"); + BOOST_CHECK_EQUAL(out.str(), "spead2: info: A test message\n"); +} + +BOOST_AUTO_TEST_SUITE_END() // logging +BOOST_AUTO_TEST_SUITE_END() // common + +}} // namespace spead2::unittest diff -Nru spead2-1.10.0/src/unittest_main.cpp spead2-2.1.0/src/unittest_main.cpp --- spead2-1.10.0/src/unittest_main.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/unittest_main.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,3 +1,25 @@ +/* Copyright 2016 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Main program for unit test framework. + */ + #define BOOST_TEST_MAIN #define BOOST_TEST_MODULE spead2 #include diff -Nru spead2-1.10.0/src/unittest_memcpy.cpp spead2-2.1.0/src/unittest_memcpy.cpp --- spead2-1.10.0/src/unittest_memcpy.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/unittest_memcpy.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,3 +1,25 @@ +/* Copyright 2016 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for accelerated memcpy. + */ + #include #include #include diff -Nru spead2-1.10.0/src/unittest_memory_allocator.cpp spead2-2.1.0/src/unittest_memory_allocator.cpp --- spead2-1.10.0/src/unittest_memory_allocator.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/unittest_memory_allocator.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,3 +1,25 @@ +/* Copyright 2016, 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for common_memory_allocator. + */ + #include #include #include @@ -31,6 +53,12 @@ ptr.reset(); } +BOOST_AUTO_TEST_CASE_TEMPLATE(out_of_memory, T, test_types) +{ + std::shared_ptr allocator = std::make_shared(); + BOOST_CHECK_THROW(allocator->allocate(SIZE_MAX - 1, nullptr), std::bad_alloc); +} + BOOST_AUTO_TEST_SUITE_END() // memory_allocator BOOST_AUTO_TEST_SUITE_END() // common diff -Nru spead2-1.10.0/src/unittest_memory_pool.cpp spead2-2.1.0/src/unittest_memory_pool.cpp --- spead2-1.10.0/src/unittest_memory_pool.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/unittest_memory_pool.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,9 +1,33 @@ +/* Copyright 2016, 2017, 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for common_memory_pool. + */ + #include #include #include #include #include #include +#include +#include #include #include #include @@ -16,6 +40,11 @@ BOOST_AUTO_TEST_SUITE(common) BOOST_AUTO_TEST_SUITE(memory_pool) +BOOST_AUTO_TEST_CASE(default_constructible) +{ + spead2::memory_pool pool; +} + // Repeatedly allocates memory from a memory pool to check that the refilling // code does not crash. BOOST_AUTO_TEST_CASE(memory_pool_refill) @@ -27,6 +56,8 @@ std::vector pointers; for (int i = 0; i < 100; i++) pointers.push_back(pool->allocate(1024 * 1024, nullptr)); + // Give the refiller time to refill completely + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } class mock_allocator : public spead2::memory_allocator diff -Nru spead2-1.10.0/src/unittest_raw_packet.cpp spead2-2.1.0/src/unittest_raw_packet.cpp --- spead2-1.10.0/src/unittest_raw_packet.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/unittest_raw_packet.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -0,0 +1,275 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for common_raw_packet. + */ + +#include +#include +#include + +namespace spead2 +{ +namespace unittest +{ + +// A packet captured with tcpdump, containing "Hello world\n" in the UDP payload +static const std::uint8_t sample_packet[] = +{ + 0x01, 0x00, 0x5e, 0x66, 0xfe, 0x01, 0x1c, 0x1b, 0x0d, 0xe0, 0xd0, 0xfd, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x28, 0xae, 0xa3, 0x40, 0x00, 0x01, 0x11, 0xd0, 0xfe, 0x0a, 0x08, 0x02, 0xb3, 0xef, 0x66, + 0xfe, 0x01, 0x87, 0x5d, 0x22, 0xb8, 0x00, 0x14, 0xe9, 0xb4, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, + 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x0a +}; +// Properties of this sample packet +static const mac_address source_mac = {{0x1c, 0x1b, 0x0d, 0xe0, 0xd0, 0xfd}}; +static const mac_address destination_mac = {{0x01, 0x00, 0x5e, 0x66, 0xfe, 0x01}}; +static const auto source_address = boost::asio::ip::address_v4::from_string("10.8.2.179"); +static const auto destination_address = boost::asio::ip::address_v4::from_string("239.102.254.1"); +static const std::uint16_t source_port = 34653; +static const std::uint16_t destination_port = 8888; +static const std::uint16_t ipv4_identification = 0xaea3; +static const std::uint16_t ipv4_checksum = 0xd0fe; +static const std::uint16_t udp_checksum = 0xe9b4; +static const std::string sample_payload = "Hello world\n"; + +struct packet_data +{ + std::array data; + + packet_data() + { + std::memcpy(data.data(), sample_packet, sizeof(sample_packet)); + } +}; + +BOOST_AUTO_TEST_SUITE(common) +BOOST_FIXTURE_TEST_SUITE(raw_packet, packet_data) + +static std::string buffer_to_string(const boost::asio::const_buffer &buffer) +{ + return std::string(boost::asio::buffer_cast(buffer), + boost::asio::buffer_cast(buffer) + boost::asio::buffer_size(buffer)); +} + +static std::string buffer_to_string(const boost::asio::mutable_buffer &buffer) +{ + return buffer_to_string(boost::asio::const_buffer(buffer)); +} + +BOOST_AUTO_TEST_CASE(multicast_mac) +{ + auto address = boost::asio::ip::address::from_string("239.202.234.100"); + mac_address result = spead2::multicast_mac(address); + mac_address expected = {{0x01, 0x00, 0x5e, 0x4a, 0xea, 0x64}}; + BOOST_CHECK_EQUAL_COLLECTIONS(result.begin(), result.end(), + expected.begin(), expected.end()); +} + +/* It's not really possible to unit test interface_mac because it interacts + * closely with the OS. But we can at least check that the error paths work. + */ +BOOST_AUTO_TEST_CASE(interface_mac_lo) +{ + auto address = boost::asio::ip::address::from_string("127.0.0.1"); + BOOST_CHECK_THROW(spead2::interface_mac(address), std::runtime_error); + address = boost::asio::ip::address::from_string("0.0.0.0"); + BOOST_CHECK_THROW(spead2::interface_mac(address), std::runtime_error); + address = boost::asio::ip::address::from_string("::1"); + BOOST_CHECK_THROW(spead2::interface_mac(address), std::runtime_error); +} + +BOOST_AUTO_TEST_CASE(packet_buffer_construct) +{ + std::uint8_t data[2]; + packet_buffer a; + packet_buffer b(data, 2); + BOOST_CHECK_EQUAL(a.data(), (std::uint8_t *) nullptr); + BOOST_CHECK_EQUAL(a.size(), 0); + BOOST_CHECK_EQUAL(b.data(), data); + BOOST_CHECK_EQUAL(b.size(), 2); +} + +BOOST_AUTO_TEST_CASE(parse_ethernet_frame) +{ + ethernet_frame frame(data.data(), data.size()); + BOOST_CHECK(frame.source_mac() == source_mac); + BOOST_CHECK(frame.destination_mac() == destination_mac); + BOOST_CHECK_EQUAL(frame.ethertype(), ipv4_packet::ethertype); +} + +// Just to get full test coverage +BOOST_AUTO_TEST_CASE(ethernet_frame_default) +{ + ethernet_frame(); +} + +BOOST_AUTO_TEST_CASE(parse_ipv4) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + + BOOST_CHECK_EQUAL(ipv4.version_ihl(), 0x45); // version 4, header length 20 + BOOST_CHECK_EQUAL(ipv4.version(), 4); + BOOST_CHECK_EQUAL(ipv4.dscp_ecn(), 0); + BOOST_CHECK_EQUAL(ipv4.total_length(), 40); + BOOST_CHECK_EQUAL(ipv4.identification(), ipv4_identification); + BOOST_CHECK_EQUAL(ipv4.flags_frag_off(), ipv4_packet::flag_do_not_fragment); + BOOST_CHECK_EQUAL(ipv4.ttl(), 1); + BOOST_CHECK_EQUAL(ipv4.protocol(), udp_packet::protocol); + BOOST_CHECK_EQUAL(ipv4.checksum(), ipv4_checksum); + BOOST_CHECK_EQUAL(ipv4.source_address(), source_address); + BOOST_CHECK_EQUAL(ipv4.destination_address(), destination_address); +} + +BOOST_AUTO_TEST_CASE(parse_udp) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + udp_packet udp = ipv4.payload_udp(); + + BOOST_CHECK_EQUAL(udp.source_port(), source_port); + BOOST_CHECK_EQUAL(udp.destination_port(), destination_port); + BOOST_CHECK_EQUAL(udp.length(), 20); + BOOST_CHECK_EQUAL(udp.checksum(), udp_checksum); + BOOST_CHECK_EQUAL(buffer_to_string(udp.payload()), sample_payload); +} + +BOOST_AUTO_TEST_CASE(ethernet_too_small) +{ + BOOST_CHECK_THROW(ethernet_frame(data.data(), 11), std::length_error); +} + +BOOST_AUTO_TEST_CASE(ipv4_too_small) +{ + ethernet_frame frame(data.data(), 30); + BOOST_CHECK_THROW(frame.payload_ipv4(), std::length_error); +} + +BOOST_AUTO_TEST_CASE(udp_too_small) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.total_length(23); + BOOST_CHECK_THROW(ipv4.payload_udp(), std::length_error); +} + +BOOST_AUTO_TEST_CASE(udp_bad_ipv4_ihl) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.version_ihl(0x44); // 16 byte header: too small + BOOST_CHECK_THROW(ipv4.payload_udp(), std::length_error); + ipv4.version_ihl(0x44); // 60 byte header: bigger than total length + BOOST_CHECK_THROW(ipv4.payload_udp(), std::length_error); +} + +BOOST_AUTO_TEST_CASE(udp_bad_ipv4_total_length) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.total_length(1); + BOOST_CHECK_THROW(ipv4.payload_udp(), std::length_error); + ipv4.total_length(100); + BOOST_CHECK_THROW(ipv4.payload_udp(), std::length_error); +} + +BOOST_AUTO_TEST_CASE(udp_bad_length) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + udp_packet udp = ipv4.payload_udp(); + udp.length(100); + BOOST_CHECK_THROW(udp.payload(), std::length_error); +} + +BOOST_AUTO_TEST_CASE(udp_from_ethernet) +{ + boost::asio::mutable_buffer payload = spead2::udp_from_ethernet(data.data(), data.size()); + BOOST_CHECK_EQUAL(buffer_to_string(payload), sample_payload); +} + +BOOST_AUTO_TEST_CASE(udp_from_ethernet_not_ipv4) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.version_ihl(0x55); // IPv4 ethertype, but header is not v4 + BOOST_CHECK_THROW(spead2::udp_from_ethernet(data.data(), data.size()), packet_type_error); +} + +BOOST_AUTO_TEST_CASE(udp_from_ethernet_not_ipv4_ethertype) +{ + ethernet_frame frame(data.data(), data.size()); + frame.ethertype(0x86DD); // IPv6 ethertype + BOOST_CHECK_THROW(spead2::udp_from_ethernet(data.data(), data.size()), packet_type_error); +} + +BOOST_AUTO_TEST_CASE(udp_from_ethernet_not_udp) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.protocol(1); // ICMP + BOOST_CHECK_THROW(spead2::udp_from_ethernet(data.data(), data.size()), packet_type_error); +} + +BOOST_AUTO_TEST_CASE(udp_from_ethernet_fragmented) +{ + ethernet_frame frame(data.data(), data.size()); + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.flags_frag_off(ipv4_packet::flag_more_fragments); + BOOST_CHECK_THROW(spead2::udp_from_ethernet(data.data(), data.size()), packet_type_error); +} + +// Build a packet from scratch and check that it matches the sample packet +BOOST_AUTO_TEST_CASE(build) +{ + std::array packet = {}; + + ethernet_frame frame(packet.data(), packet.size()); + frame.source_mac(source_mac); + frame.destination_mac(destination_mac); + frame.ethertype(ipv4_packet::ethertype); + + ipv4_packet ipv4 = frame.payload_ipv4(); + ipv4.version_ihl(0x45); // version 4, header length 20 + ipv4.total_length(40); + ipv4.identification(ipv4_identification); + ipv4.flags_frag_off(ipv4_packet::flag_do_not_fragment); + ipv4.ttl(1); + ipv4.protocol(udp_packet::protocol); + ipv4.source_address(source_address); + ipv4.destination_address(destination_address); + ipv4.update_checksum(); + + udp_packet udp = ipv4.payload_udp(); + udp.source_port(source_port); + udp.destination_port(destination_port); + udp.length(20); + udp.checksum(udp_checksum); + + boost::asio::mutable_buffer payload = udp.payload(); + boost::asio::buffer_copy(payload, boost::asio::buffer(sample_payload)); + + BOOST_CHECK_EQUAL_COLLECTIONS(data.begin(), data.end(), packet.begin(), packet.end()); +} + +BOOST_AUTO_TEST_SUITE_END() // raw_packet +BOOST_AUTO_TEST_SUITE_END() // common + +}} // namespace spead2::unittest diff -Nru spead2-1.10.0/src/unittest_recv_custom_memcpy.cpp spead2-2.1.0/src/unittest_recv_custom_memcpy.cpp --- spead2-1.10.0/src/unittest_recv_custom_memcpy.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/unittest_recv_custom_memcpy.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -0,0 +1,105 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for recv stream with custom memcpy function. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ +namespace unittest +{ + +BOOST_AUTO_TEST_SUITE(recv) +BOOST_AUTO_TEST_SUITE(custom_memcpy) + +static void reverse_memcpy(const spead2::memory_allocator::pointer &allocation, const spead2::recv::packet_header &packet) +{ + std::uint8_t *ptr = allocation.get() + (packet.heap_length - packet.payload_offset); + for (std::size_t i = 0; i < packet.payload_length; i++) + *--ptr = packet.payload[i]; +} + +/* Set up a receive stream that uses a custom memcpy to + * reverse all the payload bytes in a heap. + */ +BOOST_AUTO_TEST_CASE(test_reverse) +{ + // Create some random data to transmit + std::mt19937 engine; + std::uniform_int_distribution bytes_dist(0, 255); + std::vector data(100000); + for (auto &v : data) + v = bytes_dist(engine); + + // Set up receiver + thread_pool tp; + std::shared_ptr queue = std::make_shared(); + spead2::recv::ring_stream<> recv_stream(tp); + recv_stream.set_allow_unsized_heaps(false); + recv_stream.set_memcpy(reverse_memcpy); + recv_stream.emplace_reader(queue); + + // Set up sender and send the heap + spead2::send::inproc_stream send_stream(tp, queue); + flavour f(4, 64, 48); + spead2::send::heap send_heap(f); + spead2::send::heap stop_heap(f); + send_heap.add_item(0x1000, data.data(), data.size(), false); + stop_heap.add_end(); + send_stream.async_send_heap( + send_heap, + [&](const boost::system::error_code &ec, item_pointer_t bytes_transferred) {}); + send_stream.async_send_heap( + stop_heap, + [&](const boost::system::error_code &ec, item_pointer_t bytes_transferred) {}); + send_stream.flush(); + + // Retrieve the heap and check the content + spead2::recv::heap recv_heap = recv_stream.pop(); + bool found = false; + for (const auto &item : recv_heap.get_items()) + { + if (item.id == 0x1000) + { + BOOST_CHECK(!found); + BOOST_CHECK_EQUAL_COLLECTIONS(data.rbegin(), data.rend(), item.ptr, item.ptr + item.length); + found = true; + } + } + BOOST_CHECK(found); +} + +BOOST_AUTO_TEST_SUITE_END() // custom_memcpy +BOOST_AUTO_TEST_SUITE_END() // recv + +}} // namespace spead2::unittest diff -Nru spead2-1.10.0/src/unittest_recv_live_heap.cpp spead2-2.1.0/src/unittest_recv_live_heap.cpp --- spead2-1.10.0/src/unittest_recv_live_heap.cpp 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/src/unittest_recv_live_heap.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,3 +1,25 @@ +/* Copyright 2016, 2018 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for recv_live_heap. + */ + #include #include #include diff -Nru spead2-1.10.0/src/unittest_semaphore.cpp spead2-2.1.0/src/unittest_semaphore.cpp --- spead2-1.10.0/src/unittest_semaphore.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/unittest_semaphore.cpp 2020-02-12 12:27:05.000000000 +0000 @@ -0,0 +1,174 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for common_semaphore. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace spead2 +{ +namespace unittest +{ + +BOOST_AUTO_TEST_SUITE(common) +BOOST_AUTO_TEST_SUITE(semaphore) + +typedef boost::mpl::list< + spead2::semaphore_spin, + spead2::semaphore_pipe, +#if SPEAD2_USE_EVENTFD + spead2::semaphore_eventfd, +#endif +#if SPEAD2_USE_POSIX_SEMAPHORES + spead2::semaphore_posix +#else + spead2::semaphore // a new type only when not using posix semaphores +#endif + > semaphore_types; + +typedef boost::mpl::list< +#if SPEAD2_USE_EVENTFD + spead2::semaphore_eventfd, +#endif + spead2::semaphore_pipe> semaphore_fd_types; + +/* Try to get a semaphore, but return only if it's zero, not on an interrupted + * system call. + */ +template +static int semaphore_try_get(T &sem) +{ + while (true) + { + errno = 0; + int result = sem.try_get(); + if (result != -1 || errno != EINTR) + return result; + } +} + +/* Poll that restarts after interrupted system calls (but does not try to + * adjust the timeout to compensate). + */ +static int poll_restart(struct pollfd *fds, nfds_t nfds, int timeout) +{ + while (true) + { + int result = poll(fds, nfds, timeout); + if (result >= 0 || errno != EINTR) + return result; + } +} + +/* Gets a semaphore until it would block, to determine what value it had. + * It does not restore the previous value. + */ +template +static int semaphore_get_value(T &sem) +{ + int value = 0; + int result; + while ((result = semaphore_try_get(sem)) == 0) + value++; + BOOST_CHECK_EQUAL(result, -1); + return value; +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(single_thread, T, semaphore_types) +{ + T sem(2); + BOOST_CHECK_EQUAL(semaphore_get_value(sem), 2); + sem.put(); + BOOST_CHECK_EQUAL(semaphore_get_value(sem), 1); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(multi_thread, T, semaphore_types) +{ + const std::int64_t N = 100000; + std::vector x(N); + T sem(0); + /* Have a separate thread write to x and put a semaphore each time it + * does, and this thread get the semaphore before reading a value. This + * doesn't necessarily prove that anything works (that's basically + * impossible for multi-threading primitives), but it can show up + * failures. + */ + auto worker = [&x, &sem, N] { + for (int i = 0; i < N; i++) + { + x[i] = i; + sem.put(); + } + }; + auto future = std::async(std::launch::async, worker); + std::int64_t sum = 0; + for (int i = 0; i < N; i++) + { + semaphore_get(sem); + sum += x[i]; + } + future.get(); + BOOST_CHECK_EQUAL(sum, N * (N - 1) / 2); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(move_assign, T, semaphore_fd_types) +{ + T sem1(2); + int orig_fd = sem1.get_fd(); + T sem2; + sem2 = std::move(sem1); + BOOST_CHECK_EQUAL(sem2.get_fd(), orig_fd); + BOOST_CHECK_EQUAL(semaphore_get_value(sem2), 2); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(move_construct, T, semaphore_fd_types) +{ + T sem1(2); + int orig_fd = sem1.get_fd(); + T sem2(std::move(sem1)); + BOOST_CHECK_EQUAL(sem2.get_fd(), orig_fd); + BOOST_CHECK_EQUAL(semaphore_get_value(sem2), 2); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(poll_fd, T, semaphore_fd_types) +{ + T sem(1); + pollfd fds[1]; + std::memset(&fds, 0, sizeof(fds)); + fds[0].fd = sem.get_fd(); + fds[0].events = POLLIN; + int result = poll_restart(fds, 1, 0); + BOOST_CHECK_EQUAL(result, 1); + semaphore_get(sem); + result = poll_restart(fds, 1, 1); + BOOST_CHECK_EQUAL(result, 0); +} + +BOOST_AUTO_TEST_SUITE_END() // semaphore +BOOST_AUTO_TEST_SUITE_END() // common + +}} // namespace spead2::unittest diff -Nru spead2-1.10.0/src/unittest_send_heap.cpp spead2-2.1.0/src/unittest_send_heap.cpp --- spead2-1.10.0/src/unittest_send_heap.cpp 2018-08-23 12:32:28.000000000 +0000 +++ spead2-2.1.0/src/unittest_send_heap.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -1,3 +1,25 @@ +/* Copyright 2018 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for send_heap. + */ + #include #include diff -Nru spead2-1.10.0/src/unittest_send_streambuf.cpp spead2-2.1.0/src/unittest_send_streambuf.cpp --- spead2-1.10.0/src/unittest_send_streambuf.cpp 1970-01-01 00:00:00.000000000 +0000 +++ spead2-2.1.0/src/unittest_send_streambuf.cpp 2019-05-15 13:32:58.000000000 +0000 @@ -0,0 +1,78 @@ +/* Copyright 2019 SKA South Africa + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/** + * @file + * + * Unit tests for send_streambuf. + */ + +#include +#include +#include +#include + +namespace spead2 +{ +namespace unittest +{ + +BOOST_AUTO_TEST_SUITE(send) +BOOST_AUTO_TEST_SUITE(streambuf) + +namespace +{ + +/* streambuf that does not implement overflow(). It will thus report EOF as + * soon as it runs out of its pre-defined buffer space. + */ +class array_streambuf : public std::streambuf +{ +public: + array_streambuf(char *first, char *last) + { + setp(first, last); + } +}; + +} // anonymous namespace + +/* Send data to a streambuf that returns an EOF error + * (the success case is tested more thoroughly via Python). + */ +BOOST_AUTO_TEST_CASE(send_fail) +{ + spead2::thread_pool tp; + spead2::send::heap h; + h.add_item(0x1234, 0x5678); + std::array buffer; + array_streambuf sb(buffer.begin(), buffer.end()); + spead2::send::streambuf_stream stream(tp, sb); + std::promise> result_promise; + auto handler = [&](const boost::system::error_code &ec, std::size_t bytes_transferred) + { + result_promise.set_value(std::make_pair(ec, bytes_transferred)); + }; + stream.async_send_heap(h, handler); + auto result = result_promise.get_future().get(); + BOOST_CHECK_EQUAL(result.first, boost::asio::error::eof); + BOOST_CHECK_EQUAL(result.second, 0); +} + +BOOST_AUTO_TEST_SUITE_END() // streambuf +BOOST_AUTO_TEST_SUITE_END() // send + +}} // namespace spead2::unittest diff -Nru spead2-1.10.0/.travis/install.sh spead2-2.1.0/.travis/install.sh --- spead2-1.10.0/.travis/install.sh 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/.travis/install.sh 2020-02-12 12:27:05.000000000 +0000 @@ -10,38 +10,22 @@ echo "CC = ${CC:=gcc}" echo "CXX = ${CXX:=g++}" -if [ "$TEST_PYTHON" = "yes" ]; then - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - if [[ "$PYTHON" == "python2" ]]; then - virtualenv venv - elif [[ "$PYTHON" == "python3" ]]; then - brew update - brew upgrade python - pyvenv venv - fi - elif [[ "$PYTHON" == pypy* ]]; then - curl -fSL https://bitbucket.org/pypy/pypy/downloads/${PYTHON}-v${PYPY_VERSION}-linux64.tar.bz2 | tar -jx - PY="$PWD/$PYTHON-v${PYPY_VERSION}-linux64/bin/pypy" - if [ "$PYTHON" = "pypy3" ]; then - PY="${PY}3" # binary is pypy for pypy2 but pypy3 for pypy3 - fi - virtualenv -p $PY venv - else - virtualenv -p `which $PYTHON` venv - fi - - source venv/bin/activate - pip install -U pip setuptools wheel - pip install -r requirements.txt - if [[ "$PYTHON" == "python2" ]]; then - pip install "git+https://github.com/ska-sa/PySPEAD#egg=spead" - fi - if [ "$COVERAGE" = "yes" ]; then - pip install cpp-coveralls +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + if [[ "$PYTHON" == "python3" ]]; then + python3 --version + python3 -m venv venv fi +elif [[ "$PYTHON" == pypy* ]]; then + curl -fSL https://bitbucket.org/pypy/pypy/downloads/${PYTHON}-v${PYPY_VERSION}-linux64.tar.bz2 | tar -jx + PY="$PWD/$PYTHON-v${PYPY_VERSION}-linux64/bin/pypy3" + virtualenv -p $PY venv +else + virtualenv -p `which $PYTHON` venv fi -if [ "$NETMAP" = "yes" ]; then - git clone https://github.com/luigirizzo/netmap - git -C netmap reset --hard 454ef9c # A known good version +source venv/bin/activate +pip install -U pip setuptools wheel +pip install -r requirements.txt +if [ "$COVERAGE" = "yes" ]; then + pip install cpp-coveralls fi diff -Nru spead2-1.10.0/.travis/script.sh spead2-2.1.0/.travis/script.sh --- spead2-1.10.0/.travis/script.sh 2018-12-11 10:53:30.000000000 +0000 +++ spead2-2.1.0/.travis/script.sh 2020-02-12 12:27:05.000000000 +0000 @@ -1,25 +1,17 @@ #!/bin/bash set -e -v -if [ "$TEST_PYTHON" = "yes" ]; then - set +v - source venv/bin/activate - set -v - ./bootstrap.sh -else - ./bootstrap.sh --no-python -fi - -if [ "$NETMAP" = "yes" ]; then - export CPATH="$PWD/netmap/sys" -fi +set +v +source venv/bin/activate +set -v +./bootstrap.sh if [ "$TEST_CXX" = "yes" ]; then mkdir -p build pushd build ../configure \ - --with-netmap="${NETMAP:-no}" \ --with-recvmmsg="${RECVMMSG:-no}" \ + --with-sendmmsg="${SENDMMSG:-no}" \ --with-eventfd="${EVENTFD:-no}" \ --with-ibv="${IBV:-no}" \ --with-pcap="${PCAP:-no}" \ @@ -27,12 +19,14 @@ --disable-optimized \ CXXFLAGS=-Werror make -j4 - make -j4 check + if ! make -j4 check; then + cat src/test-suite.log + exit 1 + fi popd fi if [ "$TEST_PYTHON" = "yes" ]; then - python --version if [ "$COVERAGE" = "yes" ]; then echo '[build_ext]' > setup.cfg echo 'coverage = yes' >> setup.cfg @@ -50,4 +44,5 @@ python -c "import spead2.test.shutdown; spead2.test.shutdown.$test()" done popd + flake8 fi diff -Nru spead2-1.10.0/.travis.yml spead2-2.1.0/.travis.yml --- spead2-1.10.0/.travis.yml 2018-12-11 10:56:12.000000000 +0000 +++ spead2-2.1.0/.travis.yml 2020-02-12 12:27:05.000000000 +0000 @@ -8,55 +8,40 @@ include: - os: linux sudo: false - dist: trusty + dist: xenial env: COVERAGE=yes TEST_PYTHON=yes PYTHON=python3 CC="gcc" TEST_CXX=yes CXX="g++" - NETMAP=yes RECVMMSG=yes EVENTFD=yes IBV=yes PCAP=yes + RECVMMSG=yes SENDMMSG=yes EVENTFD=yes IBV=yes PCAP=yes - os: linux sudo: false - dist: trusty - env: TEST_CXX=yes CXX="ccache g++" RECVMMSG=yes EVENTFD=yes PCAP=yes NETMAP=yes IBV=yes + dist: xenial + env: TEST_CXX=yes CXX="ccache g++" RECVMMSG=yes SENDMMSG=yes EVENTFD=yes PCAP=yes IBV=yes - os: linux sudo: false - dist: trusty + dist: xenial env: TEST_CXX=yes CXX="ccache g++" - os: linux sudo: false - dist: trusty - env: TEST_CXX=yes CXX="ccache clang++-3.5 -Qunused-arguments" RECVMMSG=yes EVENTFD=yes PCAP=yes NETMAP=yes IBV=yes + dist: xenial + env: TEST_CXX=yes CXX="ccache clang++ -Qunused-arguments" RECVMMSG=yes SENDMMSG=yes EVENTFD=yes PCAP=yes IBV=yes - os: linux sudo: false - dist: trusty - env: TEST_CXX=yes CXX="ccache clang++-3.5 -Qunused-arguments" + dist: xenial + env: TEST_CXX=yes CXX="ccache clang++ -Qunused-arguments" - os: linux sudo: false - dist: trusty - env: TEST_PYTHON=yes PYTHON=python2 CC="ccache gcc" CXX="g++" - - os: linux - sudo: false - dist: trusty - env: TEST_PYTHON=yes PYTHON=python2 CC="clang-3.5" CXX="clang++-3.5" - - os: linux - sudo: false - dist: trusty + dist: xenial env: TEST_PYTHON=yes PYTHON=python3 CC="ccache gcc" CXX="g++" - os: linux sudo: false - dist: trusty - env: TEST_PYTHON=yes PYTHON=python3 CC="clang-3.5" CXX="clang++-3.5" - - os: linux - sudo: false - dist: trusty - env: TEST_PYTHON=yes PYTHON=pypy2 CC="ccache gcc" CXX="g++" + dist: xenial + env: TEST_PYTHON=yes PYTHON=python3 CC="clang" CXX="clang++" - os: osx - osx_image: xcode7.3 + osx_image: xcode9.4 env: TEST_CXX=yes CXX="clang++" - os: osx - osx_image: xcode7.3 - env: TEST_PYTHON=yes PYTHON=python2 CC="clang" CXX="clang++" - - os: osx - osx_image: xcode7.3 + osx_image: xcode9.4 env: TEST_PYTHON=yes PYTHON=python3 CC="clang" CXX="clang++" addons: @@ -64,14 +49,12 @@ packages: - gcc - g++ - - clang-3.5 - - libboost-system1.55-dev - - libboost-test1.55-dev - - libboost-program-options1.55-dev + - clang + - libboost-system1.58-dev + - libboost-test1.58-dev + - libboost-program-options1.58-dev - libpcap-dev - - python-dev - python3-dev - - python-pip - python3-pip - librdmacm-dev - libibverbs-dev