diff -Nru clustershell-1.8/ChangeLog clustershell-1.8.1/ChangeLog --- clustershell-1.8/ChangeLog 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/ChangeLog 2018-11-05 10:31:48.000000000 +0000 @@ -1,3 +1,22 @@ +2018-10-30 S. Thiell + + * Version 1.8.1 released. The main changes are listed below. + + * Tree: support offline gateways (#260) + + * CLI: added the following command line options (#336): + --conf to specify alternative clush.conf (clush only) + --groupsconf to specify alternative groups.conf (all CLIs) + + * NodeSet: speed-up nodeset parsing (#370) + + * EventHandler: reinstate ev_error and ev_timeout as deprecated (#377) + + * nodeset/cluset CLI: allow litteral new line in -S, so both -S "\n" + and -S $'\n' will work + + * nodeset/cluset CLI: handle multiline shell arguments in options (#394) + 2017-10-23 S. Thiell * Version 1.8 released. @@ -691,7 +710,7 @@ * Version 1.4 released. - * NodeSet.py: Add docstring for NodeSet string arithmetics (, ! & ^), + * NodeSet.py: Add docstring for NodeSet string arithmetic (, ! & ^), which is also called extended string pattern (trac ticket #127). 2010-12-14 S. Thiell @@ -1027,7 +1046,7 @@ * NodeSet.py: Added node group support with the help of the new NodeUtils module (trac ticket #41). Improved parser to support basic node/nodegroups - arithmetics (trac ticket #44). + arithmetic (trac ticket #44). * NodeUtils.py: New module that provides binding support to external node group sources (trac ticket #43). diff -Nru clustershell-1.8/conf/groups.conf.d/slurm.conf.example clustershell-1.8.1/conf/groups.conf.d/slurm.conf.example --- clustershell-1.8/conf/groups.conf.d/slurm.conf.example 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/conf/groups.conf.d/slurm.conf.example 2018-11-05 10:31:48.000000000 +0000 @@ -30,3 +30,12 @@ list: squeue -h -o "%i" -t R reverse: squeue -h -w $NODE -o "%i" cache_time: 60 + +# +# SLURM user bindings for running jobs +# +[slurmuser,su] +map: squeue -h -u $GROUP -o "%N" -t R +list: squeue -h -o "%i" -t R +reverse: squeue -h -w $NODE -o "%i" +cache_time: 60 diff -Nru clustershell-1.8/debian/changelog clustershell-1.8.1/debian/changelog --- clustershell-1.8/debian/changelog 2017-10-25 20:48:31.000000000 +0000 +++ clustershell-1.8.1/debian/changelog 2018-11-05 10:38:52.000000000 +0000 @@ -1,3 +1,16 @@ +clustershell (1.8.1-1) unstable; urgency=medium + + [ Ondřej Nový ] + * d/copyright: Use https protocol in Format field + * d/control: Remove ancient X-Python-Version field + * d/control: Remove ancient X-Python3-Version field + + [ Stéphan Gorget ] + * Debian policy 4.2.1, no changes required + * New upstream release + + -- Stéphan Gorget Mon, 05 Nov 2018 11:38:52 +0100 + clustershell (1.8-1) unstable; urgency=medium * New upstream release diff -Nru clustershell-1.8/debian/control clustershell-1.8.1/debian/control --- clustershell-1.8/debian/control 2017-10-25 20:48:31.000000000 +0000 +++ clustershell-1.8.1/debian/control 2018-11-05 10:38:52.000000000 +0000 @@ -5,11 +5,9 @@ Uploaders: Stéphan Gorget Homepage: http://cea-hpc.github.com/clustershell/ Build-Depends: debhelper (>= 9), python-all (>= 2.6.6-3~), python3-all (>= 3.4), python-setuptools (>= 0.6), python3-setuptools, dh-python -X-Python-Version: >= 2.6 -X-Python3-Version: >= 3.4 -Standards-Version: 4.1.1 -Vcs-Svn: svn://anonscm.debian.org/python-apps/packages/clustershell/trunk/ -Vcs-Browser: http://anonscm.debian.org/viewvc/python-apps/packages/clustershell/trunk/ +Standards-Version: 4.2.1 +Vcs-Git: https://salsa.debian.org/python-team/applications/clustershell.git +Vcs-Browser: https://salsa.debian.org/python-team/applications/clustershell Package: python-clustershell Architecture: all diff -Nru clustershell-1.8/debian/copyright clustershell-1.8.1/debian/copyright --- clustershell-1.8/debian/copyright 2016-12-23 11:07:41.000000000 +0000 +++ clustershell-1.8.1/debian/copyright 2018-11-05 10:38:52.000000000 +0000 @@ -1,4 +1,4 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: clustershell Upstream-Contact: Stephane Thiell Aurelien Degremont diff -Nru clustershell-1.8/debian/gbp.conf clustershell-1.8.1/debian/gbp.conf --- clustershell-1.8/debian/gbp.conf 1970-01-01 00:00:00.000000000 +0000 +++ clustershell-1.8.1/debian/gbp.conf 2018-11-05 10:38:52.000000000 +0000 @@ -0,0 +1,2 @@ +[DEFAULT] +debian-branch=debian/master diff -Nru clustershell-1.8/doc/extras/vim/syntax/groupsconf.vim clustershell-1.8.1/doc/extras/vim/syntax/groupsconf.vim --- clustershell-1.8/doc/extras/vim/syntax/groupsconf.vim 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/extras/vim/syntax/groupsconf.vim 2018-11-05 10:31:48.000000000 +0000 @@ -16,10 +16,10 @@ syn match groupsDefaultValue "\(:\|=\)\s*\w\+$"ms=s+1 contained syn match groupsColonValue "\(:\|=\).*" contained contains=groupsDefaultValue syn match groupsDefaultKey "^default\(:\|=\).*$" contains=groupsColonValue -syn match groupsGroupsDirKey "^\(groupsdir\|confdir\|autodir\)\(:\|=\)" +syn match groupsGroupsDirKey "^\(groupsdir\|confdir\|autodir\)\(:\|=\).*$" contains=groupsKeys,groupsVars " Sources -syn match groupsVars "\(\$GROUP\|\$NODE\|$SOURCE\)" contained +syn match groupsVars "\(\$GROUP\|\$NODE\|$SOURCE\|$CFGDIR\)" contained syn match groupsKeys "^\w\+\(:\|=\)"me=e-1 contained syn match groupsKeyValue "^\(map\|all\|list\|reverse\|cache_time\)\+\(:\|=\).*$" contains=groupsKeys,groupsVars @@ -47,7 +47,6 @@ HiLink groupsComment Comment HiLink groupsMainHeader Constant HiLink groupsDefaultKey Identifier - HiLink groupsGroupsDirKey Identifier HiLink groupsDefaultValue Special HiLink groupsKeys Identifier HiLink groupsVars Keyword diff -Nru clustershell-1.8/doc/man/man1/clubak.1 clustershell-1.8.1/doc/man/man1/clubak.1 --- clustershell-1.8/doc/man/man1/clubak.1 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man1/clubak.1 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH CLUBAK 1 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH CLUBAK 1 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME clubak \- format output from clush/pdsh-like output and more . @@ -71,6 +71,9 @@ .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE optional \fBgroups.conf\fP(5) group source to use .TP +.BI \-\-groupsconf\fB= FILE +use alternate config file for groups.conf(5) +.TP .B \-G\fP,\fB \-\-groupbase do not display group source prefix (always \fI@groupname\fP) .TP diff -Nru clustershell-1.8/doc/man/man1/cluset.1 clustershell-1.8.1/doc/man/man1/cluset.1 --- clustershell-1.8/doc/man/man1/cluset.1 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man1/cluset.1 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH CLUSET 1 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH CLUSET 1 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME cluset \- compute advanced cluster node set operations . @@ -61,6 +61,9 @@ .TP .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE optional \fBgroups.conf\fP(5) group source to use +.TP +.BI \-\-groupsconf\fB= FILE +use alternate config file for groups.conf(5) .UNINDENT .INDENT 0.0 .TP @@ -138,7 +141,7 @@ split result into contiguous subsets (ie. for nodeset, subsets will contain nodes with same pattern name and a contiguous range of indexes, like foobar[1\-100]; for rangeset, subsets with consists in contiguous index ranges)""" .TP .BI \-\-axis\fB= RANGESET -for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. \(aq1\(aq, \(aq1\-2\(aq, \(aq1,3\(aq, or by a single negative number meaning that the indice is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. +for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. \(aq1\(aq, \(aq1\-2\(aq, \(aq1,3\(aq, or by a single negative number meaning that the indices is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. .TP .BI \-\-pick\fB= N pick N node(s) at random in nodeset diff -Nru clustershell-1.8/doc/man/man1/clush.1 clustershell-1.8.1/doc/man/man1/clush.1 --- clustershell-1.8/doc/man/man1/clush.1 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man1/clush.1 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH CLUSH 1 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH CLUSH 1 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME clush \- execute shell commands on a cluster . @@ -66,7 +66,7 @@ .sp The \fB\-w\fP option allows you to specify remote hosts by using ClusterShell NodeSet syntax, including the node groups \fB@group\fP special syntax and the -\fBExtended Patterns\fP syntax to benefits from NodeSet basic arithmetics +\fBExtended Patterns\fP syntax to benefits from NodeSet basic arithmetic (like \fB@Agroup\e&@Bgroup\fP). See EXTENDED PATTERNS in \fBnodeset\fP(1) and also \fBgroups.conf\fP(5) for more information. .sp @@ -162,6 +162,12 @@ .B \-n\fP,\fB \-\-nostdin do not watch for possible input from stdin; this should be used when \fBclush\fP is run in the background (or in scripts). .TP +.BI \-\-groupsconf\fB= FILE +use alternate config file for groups.conf(5) +.TP +.BI \-\-conf\fB= FILE +use alternate config file for clush.conf(5) +.TP .BI \-O \ \fP,\fB \ \-\-option\fB= override any key=value \fBclush.conf\fP(5) options (repeat as needed) .UNINDENT diff -Nru clustershell-1.8/doc/man/man1/nodeset.1 clustershell-1.8.1/doc/man/man1/nodeset.1 --- clustershell-1.8/doc/man/man1/nodeset.1 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man1/nodeset.1 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH NODESET 1 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH NODESET 1 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME nodeset \- compute advanced nodeset operations . @@ -61,6 +61,9 @@ .TP .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE optional \fBgroups.conf\fP(5) group source to use +.TP +.BI \-\-groupsconf\fB= FILE +use alternate config file for groups.conf(5) .UNINDENT .INDENT 0.0 .TP @@ -138,7 +141,7 @@ split result into contiguous subsets (ie. for nodeset, subsets will contain nodes with same pattern name and a contiguous range of indexes, like foobar[1\-100]; for rangeset, subsets with consists in contiguous index ranges)""" .TP .BI \-\-axis\fB= RANGESET -for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. \(aq1\(aq, \(aq1\-2\(aq, \(aq1,3\(aq, or by a single negative number meaning that the indice is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. +for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. \(aq1\(aq, \(aq1\-2\(aq, \(aq1,3\(aq, or by a single negative number meaning that the indices is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. .TP .BI \-\-pick\fB= N pick N node(s) at random in nodeset diff -Nru clustershell-1.8/doc/man/man5/clush.conf.5 clustershell-1.8.1/doc/man/man5/clush.conf.5 --- clustershell-1.8/doc/man/man5/clush.conf.5 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man5/clush.conf.5 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH CLUSH.CONF 5 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH CLUSH.CONF 5 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME clush.conf \- Configuration file for clush . diff -Nru clustershell-1.8/doc/man/man5/groups.conf.5 clustershell-1.8.1/doc/man/man5/groups.conf.5 --- clustershell-1.8/doc/man/man5/groups.conf.5 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/man/man5/groups.conf.5 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH GROUPS.CONF 5 "2017-10-23" "1.8" "ClusterShell User Manual" +.TH GROUPS.CONF 5 "2018-10-30" "1.8.1" "ClusterShell User Manual" .SH NAME groups.conf \- Configuration file for ClusterShell node groups . diff -Nru clustershell-1.8/doc/sphinx/config.rst clustershell-1.8.1/doc/sphinx/config.rst --- clustershell-1.8/doc/sphinx/config.rst 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/config.rst 2018-11-05 10:31:48.000000000 +0000 @@ -439,6 +439,22 @@ reverse: squeue -h -w $NODE -o "%i" cache_time: 60 +The fourth section **slurmuser,su** defines a group source based on Slurm users. +Each group is based on a username and contains the nodes currently +allocated for jobs belonging to the username:: + + [slurmuser,su] + map: squeue -h -u $GROUP -o "%N" -t R + list: squeue -h -o "%i" -t R + reverse: squeue -h -w $NODE -o "%i" + cache_time: 60 + +Example of use with :ref:`clush ` to execute a command on all nodes +with running jobs of username:: + + $ clush -bw@su:username 'df -Ph /scratch' + $ clush -bw@su:username 'du -s /scratch/username' + :ref:`cache_time ` is also set to 60 seconds instead of the default (3600s) to avoid caching results in memory for too long, because this group source is likely very dynamic (this is only useful for long-running diff -Nru clustershell-1.8/doc/sphinx/conf.py clustershell-1.8.1/doc/sphinx/conf.py --- clustershell-1.8/doc/sphinx/conf.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/conf.py 2018-11-05 10:31:48.000000000 +0000 @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '1.8' +version = '1.8.1' # The full version, including alpha/beta/rc tags. -release = '1.8' +release = '1.8.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -Nru clustershell-1.8/doc/sphinx/guide/nodesets.rst clustershell-1.8.1/doc/sphinx/guide/nodesets.rst --- clustershell-1.8/doc/sphinx/guide/nodesets.rst 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/guide/nodesets.rst 2018-11-05 10:31:48.000000000 +0000 @@ -173,7 +173,7 @@ (with special character *"!"*), intersection (with special character *"&"*) and symmetric difference (with special character *"^"*) operations. String patterns are read from left to right, by proceeding any character operators -accordinately. The following example shows how you can use this feature:: +accordingly. The following example shows how you can use this feature:: >>> print NodeSet("node[10-42],node46!node10") node[11-42,46] diff -Nru clustershell-1.8/doc/sphinx/install.rst clustershell-1.8.1/doc/sphinx/install.rst --- clustershell-1.8/doc/sphinx/install.rst 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/install.rst 2018-11-05 10:31:48.000000000 +0000 @@ -127,7 +127,7 @@ openSUSE ^^^^^^^^ -ClusterShell is available in openSUSE Tumbleweed (since 2017):: +ClusterShell is available in openSUSE Tumbleweed (Factory) and Leap since 2017:: $ zypper search clustershell Loading repository data... @@ -140,7 +140,7 @@ | python3-clustershell | ClusterShell module for Python 3 | package -To install ClusterShell on openSUSE Tumbleweed (Factory), use:: +To install ClusterShell on openSUSE, use:: $ zypper install clustershell diff -Nru clustershell-1.8/doc/sphinx/release.rst clustershell-1.8.1/doc/sphinx/release.rst --- clustershell-1.8/doc/sphinx/release.rst 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/release.rst 2018-11-05 10:31:48.000000000 +0000 @@ -13,6 +13,35 @@ .. warning:: Support for Python 2.5 and below has been dropped in this version. +Version 1.8.1 +^^^^^^^^^^^^^ + +This update contains a few bug fixes and some performance improvements of the +:class:`.NodeSet` class. + +The :ref:`tree mode ` has been fixed to properly support offline +gateways. + +We added the following command line options: + +* ``--conf`` to specify alternative clush.conf (clush only) + +* ``--groupsconf`` to specify alternative groups.conf (all CLIs) + +In :class:`.EventHandler`, we reinstated :meth:`.EventHandler.ev_error`: and +:meth:`.EventHandler.ev_error`: (as deprecated) for compatibility purposes. +Please see below for more details about important :class:`.EventHandler` +changes in 1.8. + +Finally, :ref:`cluset `/:ref:`nodeset ` have been +improved by adding support for: + +* litteral new line in ``-S`` + +* multiline shell variables in options + +For more details, please have a look at `GitHub Issues for 1.8.1 milestone`_. + Main changes in 1.8 ^^^^^^^^^^^^^^^^^^^ @@ -96,7 +125,7 @@ * :meth:`.EventHandler.ev_pickup`: new ``node`` argument * :meth:`.EventHandler.ev_read`: new ``node``, ``sname`` and ``msg`` arguments -* :meth:`.EventHandler.ev_hup`: new ``rc`` argument +* :meth:`.EventHandler.ev_hup`: new ``node``, ``rc`` argument * :meth:`.EventHandler.ev_close`: new ``timedout`` argument Both old and new signatures are supported in 1.8. The old signatures will @@ -469,6 +498,7 @@ .. _GitHub Issues for 1.7.2 milestone: https://github.com/cea-hpc/clustershell/issues?utf8=%E2%9C%93&q=is%3Aissue+milestone%3A1.7.2 .. _GitHub Issues for 1.7.3 milestone: https://github.com/cea-hpc/clustershell/issues?utf8=%E2%9C%93&q=is%3Aissue+milestone%3A1.7.3 .. _GitHub Issues for 1.8 milestone: https://github.com/cea-hpc/clustershell/issues?utf8=%E2%9C%93&q=is%3Aissue+milestone%3A1.8 +.. _GitHub Issues for 1.8.1 milestone: https://github.com/cea-hpc/clustershell/issues?utf8=%E2%9C%93&q=is%3Aissue+milestone%3A1.8.1 .. _LGPL v2.1+: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html .. _CeCILL-C V1: http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html .. _xCAT: https://xcat.org/ diff -Nru clustershell-1.8/doc/sphinx/tools/clush.rst clustershell-1.8.1/doc/sphinx/tools/clush.rst --- clustershell-1.8/doc/sphinx/tools/clush.rst 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/sphinx/tools/clush.rst 2018-11-05 10:31:48.000000000 +0000 @@ -47,7 +47,7 @@ :class:`.NodeSet` syntax, including the node groups *@group* special syntax (cf. :ref:`nodeset-groupsexpr`) and the Extended String Patterns syntax (see :ref:`class-NodeSet-extended-patterns`) to benefits from :class:`.NodeSet` -basic arithmetics (like ``@Agroup&@Bgroup``). Additionally, the ``-x`` option +basic arithmetic (like ``@Agroup&@Bgroup``). Additionally, the ``-x`` option allows you to exclude nodes from remote hosts list (the same NodeSet syntax can be used here). Nodes exclusion has priority over nodes addition. diff -Nru clustershell-1.8/doc/txt/clubak.txt clustershell-1.8.1/doc/txt/clubak.txt --- clustershell-1.8/doc/txt/clubak.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/clubak.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ -------------------------------------------------- :Author: Stephane Thiell -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 1 :Manual group: ClusterShell User Manual @@ -49,6 +49,8 @@ -r, --regroup fold nodeset using node groups -s GROUPSOURCE, --groupsource=GROUPSOURCE optional ``groups.conf``\(5) group source to use +--groupsconf=FILE + use alternate config file for groups.conf(5) -G, --groupbase do not display group source prefix (always `@groupname`) -S SEPARATOR, --separator=SEPARATOR diff -Nru clustershell-1.8/doc/txt/cluset.txt clustershell-1.8.1/doc/txt/cluset.txt --- clustershell-1.8/doc/txt/cluset.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/cluset.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ -------------------------------------------- :Author: Stephane Thiell -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 1 :Manual group: ClusterShell User Manual @@ -41,6 +41,7 @@ -h, --help show this help message and exit -s GROUPSOURCE, --groupsource=GROUPSOURCE optional ``groups.conf``\(5) group source to use + --groupsconf=FILE use alternate config file for groups.conf(5) Commands: -c, --count show number of nodes in nodeset(s) @@ -76,7 +77,7 @@ return sliced off result; examples of SLICE_RANGESET are "0" for simple index selection, or "1-9/2,16" for complex rangeset selection --split=MAXSPLIT split result into a number of subsets --contiguous split result into contiguous subsets (ie. for nodeset, subsets will contain nodes with same pattern name and a contiguous range of indexes, like foobar[1-100]; for rangeset, subsets with consists in contiguous index ranges)""" - --axis=RANGESET for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. '1', '1-2', '1,3', or by a single negative number meaning that the indice is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. + --axis=RANGESET for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. '1', '1-2', '1,3', or by a single negative number meaning that the indices is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. --pick=N pick N node(s) at random in nodeset diff -Nru clustershell-1.8/doc/txt/clush.conf.txt clustershell-1.8.1/doc/txt/clush.conf.txt --- clustershell-1.8/doc/txt/clush.conf.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/clush.conf.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ ------------------------------ :Author: Stephane Thiell, -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 5 :Manual group: ClusterShell User Manual diff -Nru clustershell-1.8/doc/txt/clush.txt clustershell-1.8.1/doc/txt/clush.txt --- clustershell-1.8/doc/txt/clush.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/clush.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ ----------------------------------- :Author: Stephane Thiell -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 1 :Manual group: ClusterShell User Manual @@ -54,7 +54,7 @@ The ``-w`` option allows you to specify remote hosts by using ClusterShell NodeSet syntax, including the node groups ``@group`` special syntax and the - ``Extended Patterns`` syntax to benefits from NodeSet basic arithmetics + ``Extended Patterns`` syntax to benefits from NodeSet basic arithmetic (like ``@Agroup\&@Bgroup``). See EXTENDED PATTERNS in ``nodeset``\(1) and also ``groups.conf``\(5) for more information. @@ -131,6 +131,8 @@ -s GROUPSOURCE, --groupsource=GROUPSOURCE optional ``groups.conf``\(5) group source to use -n, --nostdin do not watch for possible input from stdin; this should be used when ``clush`` is run in the background (or in scripts). +--groupsconf=FILE use alternate config file for groups.conf(5) +--conf=FILE use alternate config file for clush.conf(5) -O , --option= override any key=value ``clush.conf``\(5) options (repeat as needed) diff -Nru clustershell-1.8/doc/txt/groups.conf.txt clustershell-1.8.1/doc/txt/groups.conf.txt --- clustershell-1.8/doc/txt/groups.conf.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/groups.conf.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ ----------------------------------------------- :Author: Stephane Thiell, -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 5 :Manual group: ClusterShell User Manual diff -Nru clustershell-1.8/doc/txt/nodeset.txt clustershell-1.8.1/doc/txt/nodeset.txt --- clustershell-1.8/doc/txt/nodeset.txt 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/doc/txt/nodeset.txt 2018-11-05 10:31:48.000000000 +0000 @@ -7,9 +7,9 @@ ----------------------------------- :Author: Stephane Thiell -:Date: 2017-10-23 +:Date: 2018-10-30 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+) -:Version: 1.8 +:Version: 1.8.1 :Manual section: 1 :Manual group: ClusterShell User Manual @@ -41,6 +41,7 @@ -h, --help show this help message and exit -s GROUPSOURCE, --groupsource=GROUPSOURCE optional ``groups.conf``\(5) group source to use + --groupsconf=FILE use alternate config file for groups.conf(5) Commands: -c, --count show number of nodes in nodeset(s) @@ -76,7 +77,7 @@ return sliced off result; examples of SLICE_RANGESET are "0" for simple index selection, or "1-9/2,16" for complex rangeset selection --split=MAXSPLIT split result into a number of subsets --contiguous split result into contiguous subsets (ie. for nodeset, subsets will contain nodes with same pattern name and a contiguous range of indexes, like foobar[1-100]; for rangeset, subsets with consists in contiguous index ranges)""" - --axis=RANGESET for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. '1', '1-2', '1,3', or by a single negative number meaning that the indice is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. + --axis=RANGESET for nD nodesets, fold along provided axis only. Axis are indexed from 1 to n and can be specified here either using the rangeset syntax, eg. '1', '1-2', '1,3', or by a single negative number meaning that the indices is counted from the end. Because some nodesets may have several different dimensions, axis indices are silently truncated to fall in the allowed range. --pick=N pick N node(s) at random in nodeset diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Clubak.py clustershell-1.8.1/lib/ClusterShell/CLI/Clubak.py --- clustershell-1.8/lib/ClusterShell/CLI/Clubak.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Clubak.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ # # Copyright (C) 2010-2012 CEA/DAM -# Copyright (C) 2017 Stephane Thiell +# Copyright (C) 2017-2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -31,9 +31,10 @@ from ClusterShell.MsgTree import MsgTree, MODE_DEFER, MODE_TRACE from ClusterShell.NodeSet import NodeSet, NodeSetParseError, std_group_resolver +from ClusterShell.NodeSet import set_std_group_resolver_config from ClusterShell.CLI.Display import Display, THREE_CHOICES -from ClusterShell.CLI.Display import sys_stdin, sys_stdout, sys_stderr +from ClusterShell.CLI.Display import sys_stdin, sys_stdout from ClusterShell.CLI.Error import GENERIC_ERRORS, handle_generic_error from ClusterShell.CLI.OptionParser import OptionParser from ClusterShell.CLI.Utils import nodeset_cmpkey @@ -94,12 +95,15 @@ # Argument management parser = OptionParser("%prog [options]") + parser.install_groupsconf_option() parser.install_display_options(verbose_options=True, separator_option=True, dshbak_compat=True, msgtree_mode=True) options = parser.parse_args()[0] + set_std_group_resolver_config(options.groupsconf) + if options.interpret_keys == THREE_CHOICES[-1]: # auto? enable_nodeset_key = None # AUTO else: diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Clush.py clustershell-1.8.1/lib/ClusterShell/CLI/Clush.py --- clustershell-1.8/lib/ClusterShell/CLI/Clush.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Clush.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ # # Copyright (C) 2007-2016 CEA/DAM -# Copyright (C) 2015-2017 Stephane Thiell +# Copyright (C) 2015-2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -59,8 +59,8 @@ from ClusterShell.Event import EventHandler from ClusterShell.MsgTree import MsgTree -from ClusterShell.NodeSet import RESOLVER_NOGROUP, std_group_resolver -from ClusterShell.NodeSet import NodeSet, NodeSetParseError +from ClusterShell.NodeSet import RESOLVER_NOGROUP, set_std_group_resolver_config +from ClusterShell.NodeSet import NodeSet, NodeSetParseError, std_group_resolver from ClusterShell.Task import Task, task_self @@ -793,7 +793,8 @@ parser.add_option("-n", "--nostdin", action="store_true", dest="nostdin", help="don't watch for possible input from stdin") - parser.install_config_options('clush.conf(5)') + parser.install_groupsconf_option() + parser.install_clush_config_options() parser.install_nodes_options() parser.install_display_options(verbose_options=True) parser.install_filecopy_options() @@ -801,10 +802,12 @@ (options, args) = parser.parse_args() + set_std_group_resolver_config(options.groupsconf) + # # Load config file and apply overrides # - config = ClushConfig(options) + config = ClushConfig(options, options.conf) # Initialize logging if config.verbosity >= VERB_DEBUG: diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Display.py clustershell-1.8.1/lib/ClusterShell/CLI/Display.py --- clustershell-1.8/lib/ClusterShell/CLI/Display.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Display.py 2018-11-05 10:31:48.000000000 +0000 @@ -81,7 +81,7 @@ # diff implies at least -b self.gather = options.gatherall or options.gather or options.diff self.progress = getattr(options, 'progress', False) # only in clush - # check parameter combinaison + # check parameter compatibility if options.diff and options.line_mode: raise ValueError("diff not supported in line_mode") self.line_mode = options.line_mode diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Error.py clustershell-1.8.1/lib/ClusterShell/CLI/Error.py --- clustershell-1.8/lib/ClusterShell/CLI/Error.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Error.py 2018-11-05 10:31:48.000000000 +0000 @@ -23,6 +23,12 @@ from __future__ import print_function +try: + import configparser +except ImportError: + # Python 2 compat + import ConfigParser as configparser + import errno import logging import os.path @@ -38,11 +44,13 @@ from ClusterShell.NodeUtils import GroupSourceNoUpcall from ClusterShell.NodeSet import NodeSetExternalError, NodeSetParseError from ClusterShell.NodeSet import RangeSetParseError +from ClusterShell.Propagation import RouteResolvingError from ClusterShell.Topology import TopologyError from ClusterShell.Worker.EngineClient import EngineClientError from ClusterShell.Worker.Worker import WorkerError -GENERIC_ERRORS = (EngineNotSupportedError, +GENERIC_ERRORS = (configparser.Error, + EngineNotSupportedError, EngineClientError, NodeSetExternalError, NodeSetParseError, @@ -52,6 +60,7 @@ GroupResolverSourceError, GroupSourceError, GroupSourceNoUpcall, + RouteResolvingError, TopologyError, TypeError, IOError, @@ -86,8 +95,10 @@ print(msgfmt % (prog, exc, exc.group_source.name), file=sys.stderr) except GroupSourceError as exc: print("%s: Group error: %s" % (prog, exc), file=sys.stderr) - except TopologyError as exc: + except (RouteResolvingError, TopologyError) as exc: print("%s: TREE MODE: %s" % (prog, exc), file=sys.stderr) + except configparser.Error as exc: + print("%s: %s" % (prog, exc), file=sys.stderr) except (TypeError, WorkerError) as exc: print("%s: %s" % (prog, exc), file=sys.stderr) except (IOError, OSError) as exc: # see PEP 3151 diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Nodeset.py clustershell-1.8.1/lib/ClusterShell/CLI/Nodeset.py --- clustershell-1.8/lib/ClusterShell/CLI/Nodeset.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Nodeset.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,6 +1,6 @@ # # Copyright (C) 2008-2016 CEA/DAM -# Copyright (C) 2015-2017 Stephane Thiell +# Copyright (C) 2015-2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -36,8 +36,8 @@ from ClusterShell.CLI.Error import GENERIC_ERRORS, handle_generic_error from ClusterShell.CLI.OptionParser import OptionParser -from ClusterShell.NodeSet import NodeSet, RangeSet -from ClusterShell.NodeSet import grouplist, std_group_resolver +from ClusterShell.NodeSet import NodeSet, RangeSet, std_group_resolver +from ClusterShell.NodeSet import grouplist, set_std_group_resolver_config from ClusterShell.NodeUtils import GroupSourceNoUpcall @@ -59,7 +59,9 @@ """Apply operations and operands from args on xset, an initial RangeSet or NodeSet.""" class_set = xset.__class__ - # Process operations + # Process operations from command arguments. + # The special argument string "-" indicates to read stdin. + # We also take care of multiline shell arguments (#394). while args: arg = args.pop(0) if arg in ("-i", "--intersection"): @@ -67,25 +69,27 @@ if val == '-': process_stdin(xset.intersection_update, class_set, autostep) else: - xset.intersection_update(class_set(val, autostep=autostep)) + xset.intersection_update(class_set.fromlist(val.splitlines(), + autostep=autostep)) elif arg in ("-x", "--exclude"): val = args.pop(0) if val == '-': process_stdin(xset.difference_update, class_set, autostep) else: - xset.difference_update(class_set(val, autostep=autostep)) + xset.difference_update(class_set.fromlist(val.splitlines(), + autostep=autostep)) elif arg in ("-X", "--xor"): val = args.pop(0) if val == '-': process_stdin(xset.symmetric_difference_update, class_set, autostep) else: - xset.symmetric_difference_update(class_set(val, - autostep=autostep)) + xset.symmetric_difference_update( + class_set.fromlist(val.splitlines(), autostep=autostep)) elif arg == '-': process_stdin(xset.update, xset.__class__, autostep) else: - xset.update(class_set(arg, autostep=autostep)) + xset.update(class_set.fromlist(arg.splitlines(), autostep=autostep)) return xset @@ -154,11 +158,13 @@ usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]" parser = OptionParser(usage) + parser.install_groupsconf_option() parser.install_nodeset_commands() parser.install_nodeset_operations() parser.install_nodeset_options() (options, args) = parser.parse_args() + set_std_group_resolver_config(options.groupsconf) group_resolver = std_group_resolver() if options.debug: @@ -258,8 +264,9 @@ if options.list > 0 or options.listall > 0: return command_list(options, xset, group_resolver) - # Interprete special characters (may raise SyntaxError) - separator = eval('\'%s\'' % options.separator, {"__builtins__":None}, {}) + # Interpret special characters (may raise SyntaxError) + separator = eval('\'\'\'%s\'\'\'' % options.separator, + {"__builtins__":None}, {}) if options.slice_rangeset: _xset = class_set() @@ -289,7 +296,7 @@ # convert to string for sample as nsiter() is slower for big # nodesets; and we assume options.pick will remain small-ish keep = random.sample(list(xset), options.pick) - # explicit class_set creation and str() convertion for RangeSet + # explicit class_set creation and str() conversion for RangeSet keep = class_set(','.join([str(x) for x in keep])) xset.intersection_update(keep) diff -Nru clustershell-1.8/lib/ClusterShell/CLI/OptionParser.py clustershell-1.8.1/lib/ClusterShell/CLI/OptionParser.py --- clustershell-1.8/lib/ClusterShell/CLI/OptionParser.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/OptionParser.py 2018-11-05 10:31:48.000000000 +0000 @@ -83,11 +83,20 @@ type="safestring", dest="groupsource", help="optional groups.conf(5) group source to use") - def install_config_options(self, filename=''): - """Install config options override""" + def install_clush_config_options(self): + """Install config options for clush""" + # config file override (--conf) + self.add_option("--conf", action="store", metavar='FILE', + help="use alternate config file for clush.conf(5)") + # config file option overrides (-O) self.add_option("-O", "--option", action="append", metavar="KEY=VALUE", dest="option", default=[], - help="override any key=value %s options" % filename) + help="override any key=value clush.conf(5) options") + + def install_groupsconf_option(self): + """Install an alternate groups.conf file option""" + self.add_option("--groupsconf", action="store", metavar='FILE', + help="use alternate config file for groups.conf(5)") def install_nodes_options(self): """Install nodes selection options""" diff -Nru clustershell-1.8/lib/ClusterShell/CLI/Utils.py clustershell-1.8.1/lib/ClusterShell/CLI/Utils.py --- clustershell-1.8/lib/ClusterShell/CLI/Utils.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/CLI/Utils.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,5 +1,6 @@ # # Copyright (C) 2010-2015 CEA/DAM +# Copyright (C) 2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -21,8 +22,6 @@ CLI utility functions """ -import sys - (KIBI, MEBI, GIBI, TEBI) = (1024.0, 1024.0 ** 2, 1024.0 ** 3, 1024.0 ** 4) diff -Nru clustershell-1.8/lib/ClusterShell/Communication.py clustershell-1.8.1/lib/ClusterShell/Communication.py --- clustershell-1.8/lib/ClusterShell/Communication.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Communication.py 2018-11-05 10:31:48.000000000 +0000 @@ -153,7 +153,7 @@ class Channel(EventHandler): """Use this event handler to establish a communication channel between to - hosts whithin the propagation tree. + hosts within the propagation tree. The endpoint's logic has to be implemented by subclassing the Channel class and overriding the start() and recv() methods. @@ -204,11 +204,10 @@ def _close(self): """close an already opened channel""" send_endtag = self.opened - # set to False before sending tag for state test purposes - self.opened = self.setup = False if send_endtag: XMLGenerator(self.worker, encoding=ENCODING).endElement('channel') self.worker.abort() + self.opened = self.setup = False def ev_start(self, worker): """connection established. Open higher level channel""" diff -Nru clustershell-1.8/lib/ClusterShell/Engine/Engine.py clustershell-1.8.1/lib/ClusterShell/Engine/Engine.py --- clustershell-1.8/lib/ClusterShell/Engine/Engine.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Engine/Engine.py 2018-11-05 10:31:48.000000000 +0000 @@ -140,7 +140,7 @@ """ Set the next firing delay in seconds for an EngineTimer object. - The optional paramater `interval' sets the firing interval + The optional parameter `interval' sets the firing interval of the timer. If not specified, the timer fires once and then is automatically invalidated. diff -Nru clustershell-1.8/lib/ClusterShell/Event.py clustershell-1.8.1/lib/ClusterShell/Event.py --- clustershell-1.8/lib/ClusterShell/Event.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Event.py 2018-11-05 10:31:48.000000000 +0000 @@ -72,6 +72,20 @@ :param msg: message """ + def ev_error(self, worker): + """ + Called to indicate that a worker has error to read on stderr from + a specific node (or key). + + [DEPRECATED] use ev_read instead and test if sname is 'stderr' + + :param worker: :class:`.Worker` object + + Available worker attributes: + * :attr:`.Worker.current_node` - node (or key) + * :attr:`.Worker.current_errmsg` - read error message + """ + def ev_written(self, worker, node, sname, size): """ Called to indicate that some writing has been done by the worker to a @@ -106,6 +120,15 @@ command return codes) """ + def ev_timeout(self, worker): + """ + Called to indicate that a worker has timed out (worker timeout only). + + [DEPRECATED] use ev_close instead and check if timedout is True + + :param worker: :class:`.Worker` object + """ + def ev_close(self, worker, timedout): """ Called to indicate that a worker has just finished. diff -Nru clustershell-1.8/lib/ClusterShell/__init__.py clustershell-1.8.1/lib/ClusterShell/__init__.py --- clustershell-1.8/lib/ClusterShell/__init__.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/__init__.py 2018-11-05 10:31:48.000000000 +0000 @@ -34,8 +34,8 @@ - ClusterShell.Task """ -__version__ = '1.8' +__version__ = '1.8.1' __version_info__ = tuple([ int(_n) for _n in __version__.split('.')]) -__date__ = '2017/10/23' +__date__ = '2018/10/30' __author__ = 'Stephane Thiell ' __url__ = 'http://clustershell.readthedocs.org/' diff -Nru clustershell-1.8/lib/ClusterShell/NodeSet.py clustershell-1.8.1/lib/ClusterShell/NodeSet.py --- clustershell-1.8/lib/ClusterShell/NodeSet.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/NodeSet.py 2018-11-05 10:31:48.000000000 +0000 @@ -120,7 +120,7 @@ otherwise it may be referenced (should be seen as an ownership transfer upon creation). - This class implements core node set arithmetics (no string parsing here). + This class implements core node set arithmetic (no string parsing here). Example: >>> nsb = NodeSetBase('node%s-ipmi', RangeSet('1-5,7'), False) @@ -772,10 +772,13 @@ """ Class that is able to transform a source into a NodeSetBase. """ - OP_CODES = { 'update': ',', - 'difference_update': '!', - 'intersection_update': '&', - 'symmetric_difference_update': '^' } + + OP_CODES = {',': 'update', + '!': 'difference_update', + '&': 'intersection_update', + '^': 'symmetric_difference_update'} + + OP_CODES_PAT = '[%s]' % re.escape(''.join(OP_CODES.keys())) BRACKET_OPEN = '[' BRACKET_CLOSE = ']' @@ -953,14 +956,11 @@ def _next_op(self, pat): """Opcode parsing subroutine.""" - op_idx = -1 - next_op_code = None - for opc, idx in [(k, pat.find(v)) - for k, v in ParsingEngine.OP_CODES.items()]: - if idx >= 0 and (op_idx < 0 or idx <= op_idx): - next_op_code = opc - op_idx = idx - return op_idx, next_op_code + mobj = re.search(ParsingEngine.OP_CODES_PAT, pat) + if mobj: + return mobj.span()[0], mobj.group() + else: + return -1, None def _scan_string_single(self, nsstr, autostep): """Single node scan, returns (pat, list of rangesets)""" @@ -998,7 +998,7 @@ def _scan_string(self, nsstr, autostep): """Parsing engine's string scanner method (iterator).""" - next_op_code = 'update' + next_op_code = ',' # if no operator, default one is to update nodeset while nsstr: # Ignore whitespace(s) for convenience nsstr = nsstr.lstrip() @@ -1072,11 +1072,11 @@ if op_idx < 0: nsstr = None else: - opc = self.OP_CODES[next_op_code] - sfx, nsstr = sfx.split(opc, 1) + sfx, nsstr = sfx.split(next_op_code, 1) # Detected character operator so right operand is mandatory if not nsstr: - msg = "missing nodeset operand with '%s' operator" % opc + msg = "missing nodeset operand with '%s' " \ + "operator" % next_op_code raise NodeSetParseError(None, msg) # Ignore whitespace(s) @@ -1093,11 +1093,11 @@ node = nsstr nsstr = None # break next time else: - opc = self.OP_CODES[next_op_code] - node, nsstr = nsstr.split(opc, 1) + node, nsstr = nsstr.split(next_op_code, 1) # Detected character operator so both operands are mandatory if not node or not nsstr: - msg = "missing nodeset operand with '%s' operator" % opc + msg = "missing nodeset operand with '%s' " \ + "operator" % next_op_code raise NodeSetParseError(node or nsstr, msg) # Check for illegal closing bracket @@ -1108,7 +1108,8 @@ node = node.rstrip() newpat, rsets = self._scan_string_single(node, autostep) - yield op_code, newpat, _rsets4nsb(rsets, autostep) + op = ParsingEngine.OP_CODES[op_code] + yield op, newpat, _rsets4nsb(rsets, autostep) def _amend_leading_digits(self, outer, inner): """Helper to get rid of leading bracket digits. @@ -1183,7 +1184,7 @@ pattern" which adds support for union (special character ","), difference ("!"), intersection ("&") and symmetric difference ("^") operations. String patterns are read from left to right, by - proceeding any character operators accordinately. + proceeding any character operators accordingly. Extended string pattern usage examples: @@ -1230,7 +1231,7 @@ is, to fold first dimension using ``[a-b]`` rangeset syntax whenever possible). Using `fold_axis` ensures that rangeset won't be folded on unspecified axis, but please note however, that using `fold_axis` may - lead to suboptimial folding, this is because NodeSet algorithms are + lead to suboptimal folding, this is because NodeSet algorithms are optimized for folding along all axis (default behavior). """ NodeSetBase.__init__(self, autostep=autostep, fold_axis=fold_axis) @@ -1571,3 +1572,18 @@ global RESOLVER_STD_GROUP RESOLVER_STD_GROUP = new_resolver or _DEF_RESOLVER_STD_GROUP +def set_std_group_resolver_config(groupsconf, illegal_chars=None): + """ + Helper to create and set std group resolver from a config file path. + + By default, the GroupResolverConfig object is created using + illegal_chars=NodeSet.ILLEGAL_GROUP_CHARS. + + This method does nothing if groupsconf is not defined. + """ + if groupsconf: + if illegal_chars is None: + illegal_chars = ILLEGAL_GROUP_CHARS + group_resolver = NodeUtils.GroupResolverConfig(groupsconf, + illegal_chars) + set_std_group_resolver(group_resolver) diff -Nru clustershell-1.8/lib/ClusterShell/Propagation.py clustershell-1.8.1/lib/ClusterShell/Propagation.py --- clustershell-1.8/lib/ClusterShell/Propagation.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Propagation.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,7 +1,7 @@ # # Copyright (C) 2010-2016 CEA/DAM # Copyright (C) 2010-2011 Henri Doreau -# Copyright (C) 2015-2017 Stephane Thiell +# Copyright (C) 2015-2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -41,6 +41,7 @@ class RouteResolvingError(Exception): """error raised on invalid conditions during routing operations""" + class PropagationTreeRouter(object): """performs routes resolving operations within a propagation tree. This object provides a next_hop method, that will look for the best @@ -392,9 +393,13 @@ self.logger.debug("ev_close gateway=%s %s", gateway, self) self.logger.debug("ev_close rc=%s", self._rc) # may be None - if self._rc: # got explicit error code - # ev_routing? - self.logger.debug("unreachable gateway %s", gateway) - worker.task.router.mark_unreachable(gateway) - self.logger.debug("worker.task.gateways=%s", worker.task.gateways) - # TODO: find best gateway, update TreeWorker counters, relaunch... + if self._rc: + self.logger.debug("error on gateway %s (setup=%s)", gateway, + self.setup) + self.task.router.mark_unreachable(gateway) + self.logger.debug("gateway %s now set as unreachable", gateway) + + if not self.setup: + # channel was not set up: we can safely repropagate commands + for mw in set(self.task.gateways[gateway][1]): + mw._relaunch(gateway) diff -Nru clustershell-1.8/lib/ClusterShell/RangeSet.py clustershell-1.8.1/lib/ClusterShell/RangeSet.py --- clustershell-1.8/lib/ClusterShell/RangeSet.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/RangeSet.py 2018-11-05 10:31:48.000000000 +0000 @@ -56,7 +56,7 @@ self.part = part class RangeSetPaddingError(RangeSetParseError): - """Raised when a fatal padding incoherency occurs""" + """Raised when a fatal padding incoherence occurs""" def __init__(self, part, msg): RangeSetParseError.__init__(self, part, "padding mismatch (%s)" % msg) @@ -315,12 +315,12 @@ return prng = None # pending range - istart = None # processing starting indice + istart = None # processing starting indices step = 0 # processing step for sli in self._contiguous_slices(): start = sli.start stop = sli.stop - unitary = (start + 1 == stop) # one indice? + unitary = (start + 1 == stop) # one indices? if istart is None: # first loop if unitary: istart = start @@ -378,7 +378,7 @@ elif not unitary: # case: broken step by contiguous range if stepped: - # yield 'range/step' by taking first indice of new range + # yield 'range/step' by taking first indices of new range yield slice(istart, i + 1, step) i += 1 else: @@ -390,7 +390,7 @@ istart = i = k = stop - 1 step = i - k k = i - # exited loop, process pending range or indice... + # exited loop, process pending range or indices... if step == 0: if prng: yield slice(*prng) @@ -497,7 +497,7 @@ """ # Return NotImplemented instead of raising TypeError, to # indicate that the comparison is not implemented with respect - # to the other type (the other comparand then gets a change to + # to the other type (the other comparand then gets a chance to # determine the result, then it falls back to object address # comparison). if not isinstance(other, RangeSet): @@ -1131,7 +1131,8 @@ if len(self._veclist) * (len(self._veclist) - 1) / 2 > max_length * 10: # *** nD full expand is preferred *** pads = self.pads() - self._veclist = [[RangeSet.fromone(i, pad=pads[axis]) + self._veclist = [[RangeSet.fromone(i, pad=pads[axis], + autostep=self.autostep) for axis, i in enumerate(tvec)] for tvec in set(self._iter())] return diff -Nru clustershell-1.8/lib/ClusterShell/Task.py clustershell-1.8.1/lib/ClusterShell/Task.py --- clustershell-1.8/lib/ClusterShell/Task.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Task.py 2018-11-05 10:31:48.000000000 +0000 @@ -352,7 +352,7 @@ def default_excepthook(self, exc_type, exc_value, tb): """Default excepthook for a newly Task. When an exception is raised and uncaught on Task thread, excepthook is called, which - is default_excepthook by default. Once excepthook overriden, + is default_excepthook by default. Once excepthook overridden, you can still call default_excepthook if needed.""" print('Exception in thread %s:' % self.thread, file=sys.stderr) traceback.print_exception(exc_type, exc_value, tb, file=sys.stderr) @@ -529,7 +529,7 @@ presence of the nodes parameter) and immediately schedules it for execution in task's runloop. So, if the task is already running (ie. called from an event handler), the command is started immediately, - assuming current execution contraintes are met (eg. fanout value). If + assuming current execution constraints are met (eg. fanout value). If the task is not running, the command is not started but scheduled for late execution. See resume() to start task runloop. @@ -1288,7 +1288,7 @@ """ Class method that blocks calling thread until all tasks have finished (from a ClusterShell point of view, for instance, - their task.resume() return). It doesn't necessarly mean that + their task.resume() return). It doesn't necessarily mean that associated threads have finished. """ Task._task_lock.acquire() @@ -1312,12 +1312,13 @@ if gwstr not in self.gateways: chan = PropagationChannel(self, gateway) logger = logging.getLogger(__name__) - logger.info("pchannel: creating new channel %s", chan) + logger.debug("pchannel: creating new channel %s", chan) # invoke gateway timeout = None # FIXME: handle timeout for gateway channels wrkcls = self.default('distant_worker') chanworker = wrkcls(gateway, command=metaworker.invoke_gateway, handler=chan, stderr=True, timeout=timeout) + chanworker._update_task_rc = False # gateway is special! define worker._fanout to not rely on the # engine's fanout, and use the special value FANOUT_UNLIMITED to # always allow registration of gateways @@ -1355,7 +1356,7 @@ chanworker, metaworkers = self.gateways[gwstr] metaworkers.remove(metaworker) if len(metaworkers) == 0: - logger.info("pchannel_release: destroying channel %s", + logger.debug("pchannel_release: destroying channel %s", chanworker.eh) chanworker.abort() # delete gateway reference diff -Nru clustershell-1.8/lib/ClusterShell/Topology.py clustershell-1.8.1/lib/ClusterShell/Topology.py --- clustershell-1.8/lib/ClusterShell/Topology.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Topology.py 2018-11-05 10:31:48.000000000 +0000 @@ -239,7 +239,7 @@ 'Source and destination nodesets overlap') def dest(self, nodeset=None): - """get the route's destination. The optionnal argument serves for + """get the route's destination. The optional argument serves for convenience and provides a way to use the method for a subset of the whole source nodeset """ @@ -297,7 +297,7 @@ return '\n'.join([str(route) for route in self._routes]) def __iter__(self): - """return an iterator over the list of rotues""" + """return an iterator over the list of routes""" return iter(self._routes) def _introduce_circular_reference(self, route): @@ -462,7 +462,7 @@ """Return a previously generated propagation tree or build it if required. As rebuilding tree can be quite expensive, once built, the propagation tree is cached. you can force a re-generation - using the optionnal `force_rebuild' parameter. + using the optional `force_rebuild' parameter. """ if self._tree is None or force_rebuild: self._tree = self.graph.to_tree(root) diff -Nru clustershell-1.8/lib/ClusterShell/Worker/EngineClient.py clustershell-1.8.1/lib/ClusterShell/Worker/EngineClient.py --- clustershell-1.8/lib/ClusterShell/Worker/EngineClient.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/EngineClient.py 2018-11-05 10:31:48.000000000 +0000 @@ -266,6 +266,8 @@ for sname in list(self.streams): self._close_stream(sname) + self.invalidate() # set self._engine to None + def _close_stream(self, sname): """ Close specific stream by name (internal, called by engine). This method @@ -318,7 +320,8 @@ wfile = self.streams[sname] if not wfile.wbuf and wfile.eof: # remove stream from engine (not directly) - self._engine.remove_stream(self, wfile) + if self._engine: + self._engine.remove_stream(self, wfile) elif len(wfile.wbuf) > 0: try: wcnt = os.write(wfile.fd, wfile.wbuf) @@ -340,7 +343,8 @@ if wfile.eof and not wfile.wbuf: self.worker._on_written(self.key, wcnt, sname) # remove stream from engine (not directly) - self._engine.remove_stream(self, wfile) + if self._engine: + self._engine.remove_stream(self, wfile) else: self._set_writing(sname) self.worker._on_written(self.key, wcnt, sname) @@ -424,9 +428,15 @@ self._engine.remove_stream(self, wfile) def abort(self): - """Abort processing any action by this client.""" - if self._engine: - self._engine.remove(self, abort=True) + """Abort processing any action by this client. + + Safe to call on an already closing or aborting client. + """ + engine = self._engine + if engine: + self.invalidate() # set self._engine to None + engine.remove(self, abort=True) + class EnginePort(EngineClient): """ @@ -512,6 +522,7 @@ self._msgq = None del self.streams['out'] del self.streams['in'] + self.invalidate() def _handle_read(self, sname): """ diff -Nru clustershell-1.8/lib/ClusterShell/Worker/Exec.py clustershell-1.8.1/lib/ClusterShell/Worker/Exec.py --- clustershell-1.8/lib/ClusterShell/Worker/Exec.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/Exec.py 2018-11-05 10:31:48.000000000 +0000 @@ -128,6 +128,7 @@ prc = self.popen.wait() self.streams.clear() + self.invalidate() if prc >= 0: self._on_nodeset_close(self.key, prc) @@ -375,6 +376,7 @@ self._close_count += 1 assert self._close_count <= len(self._clients) if self._close_count == len(self._clients) and self.eh is not None: + # also use hasattr check because ev_timeout was missing in 1.8.0 if self._has_timeout and hasattr(self.eh, 'ev_timeout'): # Legacy ev_timeout event self.eh.ev_timeout(self) diff -Nru clustershell-1.8/lib/ClusterShell/Worker/fastsubprocess.py clustershell-1.8.1/lib/ClusterShell/Worker/fastsubprocess.py --- clustershell-1.8/lib/ClusterShell/Worker/fastsubprocess.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/fastsubprocess.py 2018-11-05 10:31:48.000000000 +0000 @@ -222,7 +222,7 @@ def _get_handles(self, stdin, stdout, stderr): - """Construct and return tupel with IO objects: + """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None diff -Nru clustershell-1.8/lib/ClusterShell/Worker/Pdsh.py clustershell-1.8.1/lib/ClusterShell/Worker/Pdsh.py --- clustershell-1.8/lib/ClusterShell/Worker/Pdsh.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/Pdsh.py 2018-11-05 10:31:48.000000000 +0000 @@ -96,6 +96,7 @@ raise WorkerError("Cannot run pdsh (error %d)" % prc) self.streams.clear() + self.invalidate() if timeout: assert abort, "abort flag not set on timeout" diff -Nru clustershell-1.8/lib/ClusterShell/Worker/Popen.py clustershell-1.8.1/lib/ClusterShell/Worker/Popen.py --- clustershell-1.8/lib/ClusterShell/Worker/Popen.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/Popen.py 2018-11-05 10:31:48.000000000 +0000 @@ -77,6 +77,7 @@ prc = self.popen.wait() self.streams.clear() + self.invalidate() if prc >= 0: # filter valid rc self.rc = prc diff -Nru clustershell-1.8/lib/ClusterShell/Worker/Tree.py clustershell-1.8.1/lib/ClusterShell/Worker/Tree.py --- clustershell-1.8/lib/ClusterShell/Worker/Tree.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/Tree.py 2018-11-05 10:31:48.000000000 +0000 @@ -334,7 +334,7 @@ self._target_count += len(targets) - self.gwtargets[str(gateway)] = targets.copy() + self.gwtargets.setdefault(str(gateway), NodeSet()).add(targets) # tar commands are built here and launched on targets if reverse: @@ -360,13 +360,32 @@ self._target_count += len(targets) - self.gwtargets[str(gateway)] = targets.copy() + self.gwtargets.setdefault(str(gateway), NodeSet()).add(targets) pchan = self.task._pchannel(gateway, self) pchan.shell(nodes=targets, command=cmd, worker=self, timeout=timeout, stderr=self.stderr, gw_invoke_cmd=self.invoke_gateway, remote=self.remote) + def _relaunch(self, previous_gateway): + """Redistribute and relaunch commands on targets that were running + on previous_gateway (which is probably marked unreachable by now) + + NOTE: Relaunch is always called after failed remote execution, so + previous_gateway must be defined. However, it is not guaranteed that + the relaunch is going to be performed using gateways (that's a feature). + """ + targets = self.gwtargets[previous_gateway].copy() + self.logger.debug("_relaunch on targets %s from previous_gateway %s", + targets, previous_gateway) + + for target in targets: + self.gwtargets[previous_gateway].remove(target) + + self._check_fini(previous_gateway) + self._target_count -= len(targets) + self._launch(targets) + def _engine_clients(self): """ Access underlying engine clients. @@ -399,10 +418,10 @@ # finalize rcopy: extract tar data if self.source and self.reverse: - for node, buf in self._rcopy_bufs.items(): - tarfileobj = self._rcopy_tars[node] + for bnode, buf in self._rcopy_bufs.items(): + tarfileobj = self._rcopy_tars[bnode] if len(buf) > 0: - self.logger.debug("flushing node %s buf %d bytes", node, + self.logger.debug("flushing node %s buf %d bytes", bnode, len(buf)) tarfileobj.write(buf) tarfileobj.flush() @@ -410,11 +429,11 @@ tmptar = tarfile.open(fileobj=tarfileobj) try: self.logger.debug("%s extracting %d members in dest %s", - node, len(tmptar.getmembers()), + bnode, len(tmptar.getmembers()), self.dest) tmptar.extractall(path=self.dest) except IOError as ex: - self._on_remote_node_msgline(node, ex, 'stderr', gateway) + self._on_remote_node_msgline(bnode, ex, 'stderr', gateway) finally: tmptar.close() self._rcopy_bufs = {} @@ -462,6 +481,7 @@ if self._close_count >= self._target_count: handler = self.eh if handler: + # also use hasattr check because ev_timeout was missing in 1.8.0 if self._has_timeout and hasattr(handler, 'ev_timeout'): handler.ev_timeout(self) _eh_sigspec_invoke_compat(handler.ev_close, 2, self, diff -Nru clustershell-1.8/lib/ClusterShell/Worker/Worker.py clustershell-1.8.1/lib/ClusterShell/Worker/Worker.py --- clustershell-1.8/lib/ClusterShell/Worker/Worker.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell/Worker/Worker.py 2018-11-05 10:31:48.000000000 +0000 @@ -51,6 +51,10 @@ # Assume new signature (2.x) return method(*args) +def _eh_sigspec_ev_read_17(ev_read): + """Helper function to check whether ev_read has the old 1.7 signature.""" + return len(getfullargspec(ev_read)[0]) == 2 + class WorkerException(Exception): """Generic worker exception.""" @@ -66,14 +70,14 @@ """ Worker is an essential base class for the ClusterShell library. The goal of a worker object is to execute a common work on a single or several - targets (abstract notion) in parallel. Concret targets and also the notion - of local or distant targets are managed by Worker's subclasses (for - example, see the DistantWorker base class). + targets (abstract notion) in parallel. Concrete targets and also the + notion of local or distant targets are managed by Worker's subclasses + (for example, see the DistantWorker base class). A configured Worker object is associated to a specific ClusterShell Task, which can be seen as a single-threaded Worker supervisor. Indeed, the work to be done is executed in parallel depending on other Workers and Task's - current paramaters, like current fanout value. + current parameters, like current fanout value. ClusterShell is designed to write event-driven applications, and the Worker class is key here as Worker objects are passed as parameter of most event @@ -106,6 +110,10 @@ # cannot currently be changed afterwards. self._fanout = FANOUT_DEFAULT + # Update task rc? [private] + # TODO: to be replaced with Task Event Handlers + self._update_task_rc = True + # Parent task (once bound) self.task = None #: worker's task when scheduled or None self.started = False #: set to True when worker has started @@ -151,10 +159,11 @@ def _on_close(self, key, rc=None): """Called to generate events when the Worker is closing.""" - # rc may be None here for example when called from StreamClient - # Only update task if rc is not None. - if rc is not None: - self.task._rc_set(self, key, rc) + if self._update_task_rc: + # rc may be None here for example when called from StreamClient + # Only update task if rc is not None. + if rc is not None: + self.task._rc_set(self, key, rc) self.current_node = key self.current_rc = rc @@ -209,7 +218,10 @@ # Base actions def abort(self): - """Abort processing any action by this worker.""" + """Abort processing any action by this worker. + + Safe to call on an already closing or aborting worker. + """ raise NotImplementedError("Derived classes must implement.") def flush_buffers(self): @@ -249,19 +261,18 @@ if sname == self.SNAME_STDERR: self.current_errmsg = msg if self.eh is not None: - # check for deprecated ev_error (< 1.8) - if hasattr(self.eh, 'ev_error'): + # call old ev_error for compat (default is no-op) + if hasattr(self.eh, 'ev_error'): # missing in 1.8.0! self.eh.ev_error(self) - else: - # ev_read: ignore old signature (< 1.8) - if len(getfullargspec(self.eh.ev_read)[0]) != 2: - ### FUTURE (2.x) ### - self.eh.ev_read(self, node, sname, msg) + # /!\ NOT elif + if not _eh_sigspec_ev_read_17(self.eh.ev_read): + ### FUTURE (2.x) ### + self.eh.ev_read(self, node, sname, msg) else: self.current_msg = msg if self.eh is not None: # ev_read: check for old signature first (< 1.8) - if len(getfullargspec(self.eh.ev_read)[0]) == 2: + if _eh_sigspec_ev_read_17(self.eh.ev_read): self.eh.ev_read(self) else: ### FUTURE (2.x) ### @@ -571,20 +582,18 @@ if sname == 'stderr': self.current_errmsg = msg if self.eh is not None: - # this part is tricky to support backward compatibility... - # check for deprecated ev_error (< 1.8) - if hasattr(self.eh, 'ev_error'): + # call old ev_error for compat (default is no-op) + if hasattr(self.eh, 'ev_error'): # missing in 1.8.0! self.eh.ev_error(self) - else: - # ev_read: ignore old signature (< 1.8) - if len(getfullargspec(self.eh.ev_read)[0]) != 2: - ### FUTURE (2.x) ### - self.eh.ev_read(self, key, sname, msg) + # /!\ NOT elif + if not _eh_sigspec_ev_read_17(self.eh.ev_read): + ### FUTURE (2.x) ### + self.eh.ev_read(self, key, sname, msg) else: self.current_msg = msg if self.eh is not None: # ev_read: check for old signature first (< 1.8) - if len(getfullargspec(self.eh.ev_read)[0]) == 2: + if _eh_sigspec_ev_read_17(self.eh.ev_read): self.eh.ev_read(self) else: ### FUTURE (2.x) ### @@ -595,11 +604,15 @@ self.task._timeout_add(self, key) # trigger timeout event (deprecated in 1.8+) + # also use hasattr check because ev_timeout was missing in 1.8.0 if self.eh and hasattr(self.eh, 'ev_timeout'): self.eh.ev_timeout(self) def abort(self): - """Abort processing any action by this worker.""" + """Abort processing any action by this worker. + + Safe to call on an already closing or aborting worker. + """ self.clients[0].abort() def read(self, node=None, sname='stdout'): diff -Nru clustershell-1.8/lib/ClusterShell.egg-info/PKG-INFO clustershell-1.8.1/lib/ClusterShell.egg-info/PKG-INFO --- clustershell-1.8/lib/ClusterShell.egg-info/PKG-INFO 2017-10-25 17:29:39.000000000 +0000 +++ clustershell-1.8.1/lib/ClusterShell.egg-info/PKG-INFO 2018-11-05 10:31:48.000000000 +0000 @@ -1,12 +1,12 @@ Metadata-Version: 1.1 Name: ClusterShell -Version: 1.8 +Version: 1.8.1 Summary: ClusterShell library and tools Home-page: http://clustershell.sourceforge.net/ Author: Stephane Thiell Author-email: sthiell@stanford.edu License: LGPLv2+ -Download-URL: http://sourceforge.net/projects/clustershell/files/clustershell/1.8/ +Download-URL: http://sourceforge.net/projects/clustershell/files/clustershell/1.8.1/ Description: ClusterShell is an event-driven open source Python framework, designed to run local or distant commands in parallel on server farms or on large Linux clusters. It will take care of common issues encountered on HPC clusters, such diff -Nru clustershell-1.8/PKG-INFO clustershell-1.8.1/PKG-INFO --- clustershell-1.8/PKG-INFO 2017-10-25 17:29:39.000000000 +0000 +++ clustershell-1.8.1/PKG-INFO 2018-11-05 10:31:48.000000000 +0000 @@ -1,12 +1,12 @@ Metadata-Version: 1.1 Name: ClusterShell -Version: 1.8 +Version: 1.8.1 Summary: ClusterShell library and tools Home-page: http://clustershell.sourceforge.net/ Author: Stephane Thiell Author-email: sthiell@stanford.edu License: LGPLv2+ -Download-URL: http://sourceforge.net/projects/clustershell/files/clustershell/1.8/ +Download-URL: http://sourceforge.net/projects/clustershell/files/clustershell/1.8.1/ Description: ClusterShell is an event-driven open source Python framework, designed to run local or distant commands in parallel on server farms or on large Linux clusters. It will take care of common issues encountered on HPC clusters, such diff -Nru clustershell-1.8/setup.py clustershell-1.8.1/setup.py --- clustershell-1.8/setup.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/setup.py 2018-11-05 10:31:48.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env python # # Copyright (C) 2008-2016 CEA/DAM -# Copyright (C) 2016-2017 Stephane Thiell +# Copyright (C) 2016-2018 Stephane Thiell # # This file is part of ClusterShell. # @@ -23,14 +23,17 @@ from setuptools import setup, find_packages -if os.geteuid() == 0: - # System-wide, out-of-prefix config install (rpmbuild or pip as root) - CFGDIR = '/etc/clustershell' -else: - # User, in-prefix config install (rpmbuild or pip as user) - CFGDIR = 'etc/clustershell' +VERSION = '1.8.1' -VERSION = '1.8' +# Default CFGDIR: in-prefix config install (rpmbuild or pip as user) +CFGDIR = 'etc/clustershell' + +# Use system-wide CFGDIR instead when installing as root on Unix +try: + if os.geteuid() == 0: + CFGDIR = '/etc/clustershell' +except AttributeError: # Windows? + pass # Dependencies (for pip install) REQUIRES = ['PyYAML'] diff -Nru clustershell-1.8/tests/CLIClubakTest.py clustershell-1.8.1/tests/CLIClubakTest.py --- clustershell-1.8/tests/CLIClubakTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/CLIClubakTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -10,6 +10,9 @@ from TLib import * from ClusterShell.CLI.Clubak import main +from ClusterShell.NodeSet import set_std_group_resolver, \ + set_std_group_resolver_config + def _outfmt(*args): outfmt = "---------------\n%s\n---------------\n bar\n" @@ -166,3 +169,42 @@ b"line_mode=False gather=True tree_depth=1\n") self._clubak_t(["--diff", "-L"], b"foo1: bar\nfoo2: bar", b'', 2, b"error: option mismatch (diff not supported in line_mode)\n") + + +class CLIClubakTestGroupsConf(CLIClubakTest): + """Unit test class for testing --groupsconf option""" + + def setUp(self): + self.gconff = make_temp_file(dedent(""" + [Main] + default: global_default + + [global_default] + map: echo foo[1-2] + all: echo @foo + list: echo foo + """).encode()) + set_std_group_resolver_config(self.gconff.name) + + # passed to --groupsconf + self.custf = make_temp_file(dedent(""" + [Main] + default: custom + + [custom] + map: echo foo[1-2] + all: echo @bar + list: echo bar + """).encode()) + + def tearDown(self): + set_std_group_resolver(None) + self.gconff = None + self.custf = None + + def test_groupsconf_option(self): + """test nodeset with --groupsconf""" + # use -r (--regroup) to test group resolution + self._clubak_t(["-r"], b"foo1: bar\nfoo2: bar", _outfmt("@foo (2)")) + self._clubak_t(["--groupsconf", self.custf.name, "-r"], + b"foo1: bar\nfoo2: bar", _outfmt("@bar (2)")) diff -Nru clustershell-1.8/tests/CLIClushTest.py clustershell-1.8.1/tests/CLIClushTest.py --- clustershell-1.8/tests/CLIClushTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/CLIClushTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -13,6 +13,7 @@ import signal import sys import tempfile +from textwrap import dedent import threading import time import unittest @@ -23,6 +24,8 @@ import ClusterShell.CLI.Clush from ClusterShell.CLI.Clush import main from ClusterShell.NodeSet import NodeSet +from ClusterShell.NodeSet import set_std_group_resolver, \ + set_std_group_resolver_config from ClusterShell.Task import task_cleanup from ClusterShell.Worker.EngineClient import EngineClientNotSupportedError @@ -579,6 +582,23 @@ "-f", "1000", "-O", "fd_max=101", "echo ok"], None, b"ok\n") + def test_039_conf_option(self): + """test clush --conf option""" + custf = make_temp_file(dedent(""" + [Main] + node_count: no + """).encode()) + + # simple test that checks if "node_count:" no from custom conf file + # is taken into account + + self._clush_t(["-b", "-R", "exec", "-w", "foo[1-10]", "echo ok"], b"", + b"---------------\nfoo[1-10] (10)\n---------------\nok\n") + + self._clush_t(["--conf", custf.name, "-b", "-R", "exec", "-w", + "foo[1-10]", "echo ok"], b"", + b"---------------\nfoo[1-10]\n---------------\nok\n") + class CLIClushTest_B_StdinFailure(unittest.TestCase): """Unit test class for testing CLI/Clush.py and stdin failure""" @@ -606,3 +626,48 @@ """test clush with broken stdin""" self._clush_t(["-w", HOSTNAME, "-v", "sleep 1"], None, b"stdin: [Errno 22] Invalid argument\n", 0, b"") + + +class CLIClushTest_C_GroupsConf(unittest.TestCase): + """Unit test class for testing CLI/Clush.py with --groupsconf""" + + def setUp(self): + self.gconff = make_temp_file(dedent(""" + [Main] + default: global_default + + [global_default] + map: echo example[1-100] + all: echo @foo,@bar,@moo + list: echo foo bar moo + """).encode()) + set_std_group_resolver_config(self.gconff.name) + + # passed to --groupsconf + self.custf = make_temp_file(dedent(""" + [Main] + default: custom + + [custom] + map: echo custom[7-42] + all: echo @selene,@artemis + list: echo selene artemis + """).encode()) + + def tearDown(self): + set_std_group_resolver(None) + self.gconff = None + self.custf = None + + def _clush_t(self, args, stdin, expected_stdout, expected_rc=0, + expected_stderr=None): + CLI_main(self, main, ['clush'] + args, stdin, expected_stdout, + expected_rc, expected_stderr) + + def test_200_groupsconf_option(self): + """test clush --groupsconf""" + self._clush_t(["-R", "exec", "-w", "@foo", "-bL", "echo ok"], None, + b"example[1-100]: ok\n", 0, b"") + self._clush_t(["--groupsconf", self.custf.name, "-R", "exec", "-w", + "@foo", "-bL", "echo ok"], None, + b"custom[7-42]: ok\n", 0, b"") diff -Nru clustershell-1.8/tests/CLIConfigTest.py clustershell-1.8.1/tests/CLIConfigTest.py --- clustershell-1.8/tests/CLIConfigTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/CLIConfigTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -30,7 +30,7 @@ f.write(b"\n") parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -53,7 +53,7 @@ f.write("[Main]\n".encode()) parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -86,7 +86,7 @@ #ssh_options: -oStrictHostKeyChecking=no""").encode()) f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -125,7 +125,7 @@ f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -161,7 +161,7 @@ f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -216,7 +216,7 @@ """ % hard2).encode()) f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -248,7 +248,7 @@ verbosity: 1""").encode()) f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -282,7 +282,7 @@ verbosity: 1""").encode()) f.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args(["-f", "36", "-u", "3", "-t", "7", @@ -308,7 +308,7 @@ # This test needs installed configuration files (needed for # maximum coverage). parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) @@ -350,7 +350,7 @@ cfgfile.flush() parser = OptionParser("dummy") - parser.install_config_options() + parser.install_clush_config_options() parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) diff -Nru clustershell-1.8/tests/CLINodesetTest.py clustershell-1.8.1/tests/CLINodesetTest.py --- clustershell-1.8/tests/CLINodesetTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/CLINodesetTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -12,7 +12,8 @@ from ClusterShell.CLI.Nodeset import main from ClusterShell.NodeUtils import GroupResolverConfig -from ClusterShell.NodeSet import set_std_group_resolver +from ClusterShell.NodeSet import set_std_group_resolver, \ + set_std_group_resolver_config class CLINodesetTestBase(unittest.TestCase): @@ -83,6 +84,9 @@ self._nodeset_t(["--count", "foo[395-442]", "--intersection", "foo", "-i", "foo[1-200,245-394]"], None, b"0\n") self._nodeset_t(["--count", "foo[395-442]", "-i", "foo", "-i", "foo[0-200,245-394]"], None, b"0\n") self._nodeset_t(["--count", "foo[395-442]", "--intersection", "bar3,bar24", "-i", "foo[1-200,245-394]"], None, b"0\n") + # multiline args (#394) + self._nodeset_t(["--count", "foo[1,2]", "-i", "foo1\nfoo2"], None, b"2\n") + self._nodeset_t(["--count", "foo[1,2]", "-i", "foo1\nfoo2", "foo3\nfoo4"], None, b"4\n") def test_003_count_intersection_stdin(self): """test nodeset --count --intersection (stdin)""" @@ -113,6 +117,9 @@ self._nodeset_t(args + ["--fold", "foo[395-442]", "foo", "foo[1-200,245-394]"], None, b"foo,foo[1-200,245-442]\n") self._nodeset_t(args + ["--fold", "foo[395-442]", "foo", "foo[0-200,245-394]"], None, b"foo,foo[0-200,245-442]\n") self._nodeset_t(args + ["--fold", "foo[395-442]", "bar3,bar24", "foo[1-200,245-394]"], None, b"bar[3,24],foo[1-200,245-442]\n") + # multiline arg (#394) + self._nodeset_t(args + ["--fold", "foo3\nfoo1\nfoo2\nbar"], None, b"bar,foo[1-3]\n") + self._nodeset_t(args + ["--fold", "foo3\n\n\nfoo1\n\nfoo2\n\n"], None, b"foo[1-3]\n") # stdin self._nodeset_t(args + ["--fold"], "\n", b"\n") self._nodeset_t(args + ["--fold"], "foo\n", b"foo\n") @@ -199,6 +206,7 @@ self._nodeset_t(["--expand", "-S", ",", "foo[1-2],bar"], None, b"bar,foo1,foo2\n") self._nodeset_t(["--expand", "-S", "uuu", "foo[1-2],bar"], None, b"baruuufoo1uuufoo2\n") self._nodeset_t(["--expand", "-S", "\\n", "foo[1-2]"], None, b"foo1\nfoo2\n") + self._nodeset_t(["--expand", "-S", "\n", "foo[1-2]"], None, b"foo1\nfoo2\n") def test_009_fold_xor(self): """test nodeset --fold --xor""" @@ -210,6 +218,9 @@ self._nodeset_t(["--fold", "foo[395-442]", "-X", "foo", "-X", "foo[1-200,245-394]"], None, b"foo,foo[1-200,245-442]\n") self._nodeset_t(["--fold", "foo[395-442]", "-X", "foo", "-X", "foo[0-200,245-394]"], None, b"foo,foo[0-200,245-442]\n") self._nodeset_t(["--fold", "foo[395-442]", "-X", "bar3,bar24", "-X", "foo[1-200,245-394]"], None, b"bar[3,24],foo[1-200,245-442]\n") + # multiline args (#394) + self._nodeset_t(["--fold", "foo[1-10]", "-X", "foo5\nfoo6\nfoo7"], None, b"foo[1-4,8-10]\n") + self._nodeset_t(["--fold", "foo[1-10]", "-X", "foo5\nfoo6\nfoo7", "foo5\nfoo6"], None, b"foo[1-6,8-10]\n") def test_010_fold_xor_stdin(self): """test nodeset --fold --xor (stdin)""" @@ -239,6 +250,9 @@ # Do no change self._nodeset_t(["--fold", "foo[6-10]", "-x", "bar[0-5]"], None, b"foo[6-10]\n") self._nodeset_t(["--fold", "foo[0-10]", "foo[13-18]", "--exclude", "foo[5-10,15]"], None, b"foo[0-4,13-14,16-18]\n") + # multiline args (#394) + self._nodeset_t(["--fold", "foo[0-5]", "-x", "foo0\nfoo9\nfoo3\nfoo2\nfoo1"], None, b"foo[4-5]\n") + self._nodeset_t(["--fold", "foo[0-5]", "-x", "foo0\nfoo9\nfoo3\nfoo2\nfoo1", "foo5\nfoo6"], None, b"foo[4-6]\n") def test_012_fold_exclude_stdin(self): """test nodeset --fold --exclude (stdin)""" @@ -823,3 +837,58 @@ def test_empty_groups_conf(self): """test nodeset with empty groups.conf""" self._nodeset_t(["--list-all"], None, b"") + + +class CLINodesetMalformedGroupsConf(CLINodesetTestBase): + """Unit test class for testing malformed groups.conf""" + + def setUp(self): + self.gconff = make_temp_file(b"[Main") + set_std_group_resolver(GroupResolverConfig(self.gconff.name)) + + def tearDown(self): + set_std_group_resolver(None) + self.gconff = None + + def test_malformed_groups_conf(self): + """test nodeset with malformed groups.conf""" + self._nodeset_t(["--list-all"], None, b"", 1, b"'[Main'\n") + + +class CLINodesetGroupsConfOption(CLINodesetTestBase): + """Unit test class for testing --groupsconf option""" + + def setUp(self): + self.gconff = make_temp_file(dedent(""" + [Main] + default: global_default + + [global_default] + map: echo example[1-100] + all: echo @foo,@bar,@moo + list: echo foo bar moo + """).encode()) + set_std_group_resolver_config(self.gconff.name) + + # passed to --groupsconf + self.custf = make_temp_file(dedent(""" + [Main] + default: custom + + [custom] + map: echo custom[7-42] + all: echo @selene,@artemis + list: echo selene artemis + """).encode()) + + def tearDown(self): + set_std_group_resolver(None) + self.gconff = None + self.custf = None + + def test_groupsconf_option(self): + """test nodeset with --groupsconf""" + self._nodeset_t(["--list-all"], None, b"@bar\n@foo\n@moo\n") + self._nodeset_t(["-f", "@foo"], None, b"example[1-100]\n") + self._nodeset_t(["--groupsconf", self.custf.name, "--list-all"], None, b"@artemis\n@selene\n") + self._nodeset_t(["--groupsconf", self.custf.name, "-f", "@artemis"], None, b"custom[7-42]\n") diff -Nru clustershell-1.8/tests/NodeSetTest.py clustershell-1.8.1/tests/NodeSetTest.py --- clustershell-1.8/tests/NodeSetTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/NodeSetTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -2671,11 +2671,11 @@ self.assertEqual(len(n1), 16) # reverse - n1.fold_axis = [-1] # first indice from the end + n1.fold_axis = [-1] # first indices from the end self.assertEqual(str(n1), "da1c[1-2],da2c[1-2],ln[0-1],master,slave,x1y1z[1-2],x2y1z[1-2],x1y2z[1-2],x2y2z[1-2]") self.assertEqual(len(n1), 16) - n1.fold_axis = [-2] # second indice from the end + n1.fold_axis = [-2] # second indices from the end self.assertEqual(str(n1), "da[1-2]c1,da[1-2]c2,ln0,ln1,master,slave,x1y[1-2]z1,x2y[1-2]z1,x1y[1-2]z2,x2y[1-2]z2") self.assertEqual(len(n1), 16) diff -Nru clustershell-1.8/tests/RangeSetNDTest.py clustershell-1.8.1/tests/RangeSetNDTest.py --- clustershell-1.8/tests/RangeSetNDTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/RangeSetNDTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -158,6 +158,11 @@ "0-2; 1-2; 0-4\n3; 1-2; 0-5\n", 42) self._testRS([["0-2", "1-2", "0-4"], ["1-3", "1-3", "0-4"]], "1-2; 1-3; 0-4\n0,3; 1-2; 0-4\n3; 3; 0-4\n", 55) + + # triggers full expand heuristic + veclist = [item for x in range(0, 22, 2) for item in [(x,0), (x,1)]] + self._testRS(veclist, "0-20/2; 0-1\n", 22) + # the following test triggers folding loop protection self._testRS([["0-100", "50-200"], ["2-101", "49"]], "0-100; 50-200\n2-101; 49\n", 15351) diff -Nru clustershell-1.8/tests/StreamWorkerTest.py clustershell-1.8.1/tests/StreamWorkerTest.py --- clustershell-1.8/tests/StreamWorkerTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/StreamWorkerTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -219,6 +219,7 @@ self.check_written += 1 self.testcase.assertEqual(os.read(self.rfd, 1024), b"initial") worker.abort() + worker.abort() # safe but no effect rfd, wfd = os.pipe() @@ -248,6 +249,7 @@ self.check_written += 1 self.testcase.assertEqual(os.read(self.rfd, 1024), b"initial") worker.abort() + worker.abort() # safe but no effect rfd, wfd = os.pipe() @@ -299,3 +301,31 @@ self.run_worker(worker) self.assertEqual(hdlr.check_hup, 1) self.assertEqual(hdlr.check_written, 1) + + def test_009_worker_abort_on_close(self): + """test StreamWorker abort() on closing worker""" + + class TestH(EventHandler): + def __init__(self, testcase, rfd): + self.testcase = testcase + self.rfd = rfd + self.check_close = 0 + + def ev_close(self, worker, timedout): + self.check_close += 1 + self.testcase.assertFalse(timedout) + os.close(self.rfd) + worker.abort() + worker.abort() # safe but no effect + + rfd, wfd = os.pipe() + + hdlr = TestH(self, rfd) + worker = StreamWorker(handler=hdlr) + + worker.set_writer("test", wfd) # closefd=True + worker.write(b"initial", "test") + worker.set_write_eof() + + self.run_worker(worker) + self.assertEqual(hdlr.check_close, 1) diff -Nru clustershell-1.8/tests/TaskDistantMixin.py clustershell-1.8.1/tests/TaskDistantMixin.py --- clustershell-1.8/tests/TaskDistantMixin.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/TaskDistantMixin.py 2018-11-05 10:31:48.000000000 +0000 @@ -612,6 +612,7 @@ self.testtimer = False def ev_timer(self, timer): self.ext_worker.abort() + self.ext_worker.abort() # safe but no effect self.testtimer = True aot = AbortOnTimer(task.shell("sleep 10", nodes=HOSTNAME)) diff -Nru clustershell-1.8/tests/TaskLocalMixin.py clustershell-1.8.1/tests/TaskLocalMixin.py --- clustershell-1.8/tests/TaskLocalMixin.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/TaskLocalMixin.py 2018-11-05 10:31:48.000000000 +0000 @@ -827,6 +827,7 @@ self.testtimer = False def ev_timer(self, timer): self.ext_worker.abort() + self.ext_worker.abort() # safe but no effect self.testtimer = True aot = AbortOnTimer(task.shell("sleep 10")) diff -Nru clustershell-1.8/tests/TreeGatewayTest.py clustershell-1.8.1/tests/TreeGatewayTest.py --- clustershell-1.8/tests/TreeGatewayTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/TreeGatewayTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -157,8 +157,6 @@ self.channel_send_stop() self.recvxml(EndMessage) - self.assertEqual(self.chan.opened, False) - self.assertEqual(self.chan.setup, False) # ending tag should abort gateway worker without delay self.gateway.wait() self.gateway.close() @@ -179,8 +177,6 @@ # gateway should terminate channel session msg = self.recvxml(EndMessage) - self.assertEqual(self.chan.opened, False) - self.assertEqual(self.chan.setup, False) self.gateway.wait() self.gateway.close() @@ -217,10 +213,6 @@ else: self.recvxml() - # flags should be reset - self.assertEqual(self.chan.opened, False) - self.assertEqual(self.chan.setup, False) - # gateway task should exit properly self.gateway.wait() self.gateway.close() diff -Nru clustershell-1.8/tests/TreeTaskTest.py clustershell-1.8.1/tests/TreeTaskTest.py --- clustershell-1.8/tests/TreeTaskTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/TreeTaskTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -7,6 +7,7 @@ from textwrap import dedent import unittest +from ClusterShell.Propagation import RouteResolvingError from ClusterShell.Task import task_self from ClusterShell.Topology import TopologyError @@ -33,14 +34,10 @@ task = task_self() task.set_default("auto_tree", True) task.TOPOLOGY_CONFIGS = [topofile.name] - task.run("/bin/hostname", nodes="dummy-node", stderr=True) - # FIXME gateway errors are not yet being handled correctly - self.assertEqual(task.max_retcode(), 255) - # XXX correct results would be: - #self.assertEqual(task.max_retcode(), None) - #expected = "Name or service not known" - #if not task.node_error("dummy-node").endswith(expected): - # self.assertEqual(task.node_error("dummy-node"), expected) + + self.assertRaises(RouteResolvingError, task.run, "/bin/hostname", + nodes="dummy-node", stderr=True) + self.assertEqual(task.max_retcode(), None) def test_shell_auto_tree_noconf(self): """test task shell auto tree [no topology.conf]""" diff -Nru clustershell-1.8/tests/WorkerExecTest.py clustershell-1.8.1/tests/WorkerExecTest.py --- clustershell-1.8/tests/WorkerExecTest.py 2017-10-25 16:42:25.000000000 +0000 +++ clustershell-1.8.1/tests/WorkerExecTest.py 2018-11-05 10:31:48.000000000 +0000 @@ -8,6 +8,7 @@ from TLib import HOSTNAME, make_temp_file, make_temp_filename, make_temp_dir +from ClusterShell.Event import EventHandler from ClusterShell.Worker.Exec import ExecWorker, WorkerError from ClusterShell.Task import task_self @@ -150,3 +151,29 @@ stderr=True, reverse=True) finally: os.rmdir(dstbasedir) + + def test_abort_on_read(self): + """test ExecWorker.abort() on read""" + + class TestH(EventHandler): + def ev_read(self, worker): + worker.abort() + worker.abort() # safe but no effect + + self.execw(nodes='localhost', handler=TestH(), + command="echo ok; tail -f /dev/null") + self.assertEqual(task_self().max_retcode(), None) + self.assertEqual(task_self().node_buffer('localhost'), b'ok') + + def test_abort_on_close(self): + """test ExecWorker.abort() on close""" + + class TestH(EventHandler): + def ev_close(self, worker, timedout): + worker.abort() + worker.abort() # safe but no effect + + self.execw(nodes='localhost', handler=TestH(), + command="echo ok; sleep .1") + self.assertEqual(task_self().max_retcode(), 0) + self.assertEqual(task_self().node_buffer('localhost'), b'ok')