diff -Nru graypy-1.1.3/debian/changelog graypy-2.1.0/debian/changelog --- graypy-1.1.3/debian/changelog 2019-07-23 11:13:34.000000000 +0000 +++ graypy-2.1.0/debian/changelog 2021-01-06 14:04:46.000000000 +0000 @@ -1,3 +1,22 @@ +graypy (2.1.0-1) unstable; urgency=medium + + [ Ondřej Nový ] + * d/control: Update Vcs-* fields with new Debian Python Team Salsa + layout. + + [ Debian Janitor ] + * Set upstream metadata fields: Bug-Database, Bug-Submit, Repository, + Repository-Browse. + + [ Benjamin Drung ] + * New upstream release. + * Update standards version to 4.5.1, no changes needed. + * Switch to debhelper 13 + * Set Rules-Requires-Root: no + * Upgrade debian/watch to version 4 + + -- Benjamin Drung Wed, 06 Jan 2021 15:04:46 +0100 + graypy (1.1.3-2) unstable; urgency=medium * Drop unused Python 2 version diff -Nru graypy-1.1.3/debian/control graypy-2.1.0/debian/control --- graypy-1.1.3/debian/control 2019-07-23 11:11:36.000000000 +0000 +++ graypy-2.1.0/debian/control 2021-01-06 14:01:22.000000000 +0000 @@ -2,7 +2,7 @@ Maintainer: Benjamin Drung Section: python Priority: optional -Build-Depends: debhelper-compat (= 12), +Build-Depends: debhelper-compat (= 13), dh-python, python3-all, python3-amqplib, @@ -10,10 +10,11 @@ python3-pytest, python3-requests, python3-setuptools -Standards-Version: 4.4.0 +Standards-Version: 4.5.1 Homepage: https://github.com/severb/graypy -Vcs-Browser: https://salsa.debian.org/python-team/modules/graypy -Vcs-Git: https://salsa.debian.org/python-team/modules/graypy.git +Rules-Requires-Root: no +Vcs-Browser: https://salsa.debian.org/python-team/packages/graypy +Vcs-Git: https://salsa.debian.org/python-team/packages/graypy.git Package: python3-graypy Architecture: all diff -Nru graypy-1.1.3/debian/upstream/metadata graypy-2.1.0/debian/upstream/metadata --- graypy-1.1.3/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/debian/upstream/metadata 2021-01-06 13:58:20.000000000 +0000 @@ -0,0 +1,5 @@ +--- +Bug-Database: https://github.com/severb/graypy/issues +Bug-Submit: https://github.com/severb/graypy/issues/new +Repository: https://github.com/severb/graypy.git +Repository-Browse: https://github.com/severb/graypy diff -Nru graypy-1.1.3/debian/watch graypy-2.1.0/debian/watch --- graypy-1.1.3/debian/watch 2016-12-23 10:26:16.000000000 +0000 +++ graypy-2.1.0/debian/watch 2021-01-06 14:03:59.000000000 +0000 @@ -1,3 +1,3 @@ -version=3 +version=4 opts=uversionmangle=s/(rc|a|b|c)/~$1/ \ https://pypi.debian.net/graypy/graypy-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff -Nru graypy-1.1.3/graypy/handler.py graypy-2.1.0/graypy/handler.py --- graypy-1.1.3/graypy/handler.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/graypy/handler.py 2019-09-30 22:34:35.000000000 +0000 @@ -3,6 +3,7 @@ """Logging Handlers that send messages in Graylog Extended Log Format (GELF)""" +import warnings import abc import datetime import json @@ -30,7 +31,7 @@ if sys.version_info >= (3, 4): # check if python3.4+ ABC = abc.ABC else: - ABC = abc.ABCMeta(str('ABC'), (), {}) + ABC = abc.ABCMeta(str("ABC"), (), {}) try: import httplib @@ -45,61 +46,61 @@ logging.DEBUG: 7, } +GELF_MAX_CHUNK_NUMBER = 128 -class BaseGELFHandler(logging.Handler, ABC): - """Abstract class noting the basic components of a GLEFHandler""" - - def __init__(self, chunk_size=WAN_CHUNK, debugging_fields=True, - extra_fields=True, fqdn=False, localname=None, facility=None, - level_names=False, compress=True): - """Initialize the BaseGELFHandler. - :param chunk_size: Message chunk size. Messages larger than this - size will be sent to Graylog in multiple chunks. Defaults to - ``WAN_CHUNK=1420``. - :type chunk_size: int +class BaseGELFHandler(logging.Handler, ABC): + """Abstract class defining the basic functionality of converting a + :obj:`logging.LogRecord` into a GELF log. Provides the boilerplate for + all GELF handlers defined within graypy.""" + + def __init__( + self, + debugging_fields=True, + extra_fields=True, + fqdn=False, + localname=None, + facility=None, + level_names=False, + compress=True, + ): + """Initialize the BaseGELFHandler :param debugging_fields: If :obj:`True` add debug fields from the - log record into the GELF logs to be sent to Graylog - (:obj:`True` by default). + log record into the GELF logs to be sent to Graylog. :type debugging_fields: bool :param extra_fields: If :obj:`True` add extra fields from the log - record into the GELF logs to be sent to Graylog - (:obj:`True` by default). + record into the GELF logs to be sent to Graylog. :type extra_fields: bool :param fqdn: If :obj:`True` use the fully qualified domain name of - localhost to populate the ``host`` GELF field - (:obj:`False` by default). + localhost to populate the ``host`` GELF field. :type fqdn: bool - :param localname: If ``fqdn`` is :obj:`False` and ``localname`` is - specified, used the specified hostname to populate the - ``host`` GELF field. + :param localname: If specified and ``fqdn`` is :obj:`False`, use the + specified hostname to populate the ``host`` GELF field. :type localname: str or None :param facility: If specified, replace the ``facility`` GELF field - with the specified value. Additionally, the LogRecord.name will - used populate the ``_logger`` GELF field. + with the specified value. Also add a additional ``_logger`` + GELF field containing the ``LogRecord.name``. :type facility: str - :param level_names: If :obj:`True` use string error level names - instead of numerical values (:obj:`False` by default). + :param level_names: If :obj:`True` use python logging error level name + strings instead of syslog numerical values. :type level_names: bool :param compress: If :obj:`True` compress the GELF message before - sending it to the server (:obj:`True` by default). + sending it to the Graylog server. :type compress: bool """ logging.Handler.__init__(self) self.debugging_fields = debugging_fields self.extra_fields = extra_fields - self.chunk_size = chunk_size if fqdn and localname: - raise ValueError( - "cannot specify 'fqdn' and 'localname' arguments together") + raise ValueError("cannot specify 'fqdn' and 'localname' arguments together") self.fqdn = fqdn self.localname = localname @@ -108,14 +109,13 @@ self.compress = compress def makePickle(self, record): - """Convert a :class:`logging.LogRecord` into a bytes object - representing a GELF log + """Convert a :class:`logging.LogRecord` into bytes representing + a GELF log - :param record: :class:`logging.LogRecord` to convert into a - Graylog GELF log. + :param record: :class:`logging.LogRecord` to convert into a GELF log. :type record: logging.LogRecord - :return: A bytes object representing a GELF log. + :return: bytes representing a GELF log. :rtype: bytes """ gelf_dict = self._make_gelf_dict(record) @@ -124,24 +124,25 @@ return pickle def _make_gelf_dict(self, record): - """Create a dictionary representing a Graylog GELF log from a + """Create a dictionary representing a GELF log from a python :class:`logging.LogRecord` - :param record: :class:`logging.LogRecord` to create a Graylog GELF - log from. + :param record: :class:`logging.LogRecord` to create a GELF log from. :type record: logging.LogRecord - :return: dictionary representing a Graylog GELF log. + :return: Dictionary representing a GELF log. :rtype: dict """ # construct the base GELF format gelf_dict = { - 'version': "1.0", - 'host': BaseGELFHandler._resolve_host(self.fqdn, self.localname), - 'short_message': self.formatter.format(record) if self.formatter else record.getMessage(), - 'timestamp': record.created, - 'level': SYSLOG_LEVELS.get(record.levelno, record.levelno), - 'facility': self.facility or record.name, + "version": "1.0", + "host": self._resolve_host(self.fqdn, self.localname), + "short_message": self.formatter.format(record) + if self.formatter + else record.getMessage(), + "timestamp": record.created, + "level": SYSLOG_LEVELS.get(record.levelno, record.levelno), + "facility": self.facility or record.name, } # add in specified optional extras @@ -162,21 +163,23 @@ the logging level via the string error level names instead of numerical values - :param gelf_dict: dictionary representation of a GELF log. + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict :param record: :class:`logging.LogRecord` to extract a logging level from to insert into the given ``gelf_dict``. :type record: logging.LogRecord """ - gelf_dict['level_name'] = logging.getLevelName(record.levelno) + gelf_dict["level_name"] = logging.getLevelName(record.levelno) @staticmethod def _set_custom_facility(gelf_dict, facility_value, record): """Set the ``gelf_dict``'s ``facility`` field to the specified value - also add the the extra ``_logger`` field containing the LogRecord.name - :param gelf_dict: dictionary representation of a GELF log. + Also add a additional ``_logger`` field containing the + ``LogRecord.name``. + + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict :param facility_value: Value to set as the ``gelf_dict``'s @@ -188,14 +191,14 @@ field. :type record: logging.LogRecord """ - gelf_dict.update({"facility": facility_value, '_logger': record.name}) + gelf_dict.update({"facility": facility_value, "_logger": record.name}) @staticmethod def _add_full_message(gelf_dict, record): """Add the ``full_message`` field to the ``gelf_dict`` if any traceback information exists within the logging record - :param gelf_dict: dictionary representation of a GELF log. + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict :param record: :class:`logging.LogRecord` to extract a full @@ -206,10 +209,9 @@ full_message = None # format exception information if present if record.exc_info: - full_message = '\n'.join( - traceback.format_exception(*record.exc_info)) + full_message = "\n".join(traceback.format_exception(*record.exc_info)) # use pre-formatted exception information in cases where the primary - # exception information was removed, eg. for LogRecord serialization + # exception information was removed, e.g. for LogRecord serialization if record.exc_text: full_message = record.exc_text if full_message: @@ -226,7 +228,7 @@ :param localname: Use specified hostname as the ``host`` GELF field. :type localname: str or None - :return: String value representing the ``host`` GELF field. + :return: String representing the ``host`` GELF field. :rtype: str """ if fqdn: @@ -239,24 +241,26 @@ def _add_debugging_fields(gelf_dict, record): """Add debugging fields to the given ``gelf_dict`` - :param gelf_dict: dictionary representation of a GELF log. + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict :param record: :class:`logging.LogRecord` to extract debugging fields from to insert into the given ``gelf_dict``. :type record: logging.LogRecord """ - gelf_dict.update({ - 'file': record.pathname, - 'line': record.lineno, - '_function': record.funcName, - '_pid': record.process, - '_thread_name': record.threadName, - }) + gelf_dict.update( + { + "file": record.pathname, + "line": record.lineno, + "_function": record.funcName, + "_pid": record.process, + "_thread_name": record.threadName, + } + ) # record.processName was added in Python 2.6.2 - pn = getattr(record, 'processName', None) + pn = getattr(record, "processName", None) if pn is not None: - gelf_dict['_process_name'] = pn + gelf_dict["_process_name"] = pn @staticmethod def _add_extra_fields(gelf_dict, record): @@ -265,7 +269,7 @@ However, this does not add additional fields in to ``message_dict`` that are either duplicated from standard :class:`logging.LogRecord` attributes, duplicated from the python logging module source - (e.g. ``exc_text``), or violate GLEF format (i.e. ``id``). + (e.g. ``exc_text``), or violate GELF format (i.e. ``id``). .. seealso:: @@ -274,80 +278,94 @@ http://docs.python.org/library/logging.html#logrecord-attributes - :param gelf_dict: dictionary representation of a GELF log. + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict :param record: :class:`logging.LogRecord` to extract extra fields from to insert into the given ``gelf_dict``. :type record: logging.LogRecord """ - # skip_list is used to filter additional fields in a log message. skip_list = ( - 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', - 'funcName', 'id', 'levelname', 'levelno', 'lineno', 'module', - 'msecs', 'message', 'msg', 'name', 'pathname', 'process', - 'processName', 'relativeCreated', 'thread', 'threadName') + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "id", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "thread", + "threadName", + ) for key, value in record.__dict__.items(): - if key not in skip_list and not key.startswith('_'): - gelf_dict['_%s' % key] = value + if key not in skip_list and not key.startswith("_"): + gelf_dict["_%s" % key] = value - @staticmethod - def _pack_gelf_dict(gelf_dict): - """Convert a given ``gelf_dict`` to a JSON-encoded string, thus, + @classmethod + def _pack_gelf_dict(cls, gelf_dict): + """Convert a given ``gelf_dict`` into JSON-encoded UTF-8 bytes, thus, creating an uncompressed GELF log ready for consumption by Graylog. Since we cannot be 100% sure of what is contained in the ``gelf_dict`` we have to do some sanitation. - :param gelf_dict: dictionary representation of a GELF log. + :param gelf_dict: Dictionary representing a GELF log. :type gelf_dict: dict - :return: A prepped JSON-encoded GELF log as a bytes string - encoded in UTF-8. + :return: Bytes representing a uncompressed GELF log. :rtype: bytes """ - gelf_dict = BaseGELFHandler._sanitize_to_unicode(gelf_dict) - packed = json.dumps( - gelf_dict, - separators=',:', - default=BaseGELFHandler._object_to_json - ) - return packed.encode('utf-8') + gelf_dict = cls._sanitize_to_unicode(gelf_dict) + packed = json.dumps(gelf_dict, separators=",:", default=cls._object_to_json) + return packed.encode("utf-8") - @staticmethod - def _sanitize_to_unicode(obj): + @classmethod + def _sanitize_to_unicode(cls, obj): """Convert all strings records of the object to unicode - :param obj: object to sanitize to unicode. + :param obj: Object to sanitize to unicode. :type obj: object - :return: Unicode string representation of the given object. + :return: Unicode string representing the given object. :rtype: str """ if isinstance(obj, dict): - return dict((BaseGELFHandler._sanitize_to_unicode(k), BaseGELFHandler._sanitize_to_unicode(v)) for k, v in obj.items()) + return dict( + (cls._sanitize_to_unicode(k), cls._sanitize_to_unicode(v)) + for k, v in obj.items() + ) if isinstance(obj, (list, tuple)): - return obj.__class__([BaseGELFHandler._sanitize_to_unicode(i) for i in obj]) + return obj.__class__([cls._sanitize_to_unicode(i) for i in obj]) if isinstance(obj, data): - obj = obj.decode('utf-8', errors='replace') + obj = obj.decode("utf-8", errors="replace") return obj @staticmethod def _object_to_json(obj): """Convert objects that cannot be natively serialized into JSON - into their string representation + into their string representation (for later JSON serialization). - For datetime based objects convert them into their ISO formatted - string as specified by :meth:`datetime.datetime.isoformat`. + :class:`datetime.datetime` based objects will be converted into a + ISO formatted timestamp string. - :param obj: object to convert into a JSON via getting its string - representation. + :param obj: Object to convert into a string representation. :type obj: object - :return: String value representing the given object ready to be - encoded into a JSON. + :return: String representing the given object. :rtype: str """ if isinstance(obj, datetime.datetime): @@ -355,73 +373,307 @@ return repr(obj) +class BaseGELFChunker(object): + """Base UDP GELF message chunker + + .. warning:: + This will silently drop chunk overflowing GELF messages. + (i.e. GELF messages that consist of more than 128 chunks) + + .. note:: + UDP GELF message chunking is only supported for the + :class:`.handler.GELFUDPHandler`. + """ + + def __init__(self, chunk_size=WAN_CHUNK): + """Initialize the BaseGELFChunker. + + :param chunk_size: Message chunk size. Messages larger than this + size should be sent to Graylog in multiple chunks. + :type chunk_size: int + """ + self.chunk_size = chunk_size + + def _message_chunk_number(self, message): + """Get the number of chunks a GELF message requires + + :return: Number of chunks the specified GELF message requires. + :rtype: int + """ + return int(math.ceil(len(message) * 1.0 / self.chunk_size)) + + @staticmethod + def _encode(message_id, chunk_seq, total_chunks, chunk): + return b"".join( + [ + b"\x1e\x0f", + struct.pack("Q", message_id), + struct.pack("B", chunk_seq), + struct.pack("B", total_chunks), + chunk, + ] + ) + + def _gen_gelf_chunks(self, message): + """Generate and iter chunks for a GELF message + + :param message: GELF message to generate and iter chunks for. + :type; bytes + + :return: Iterator of the chunks of a GELF message. + :rtype: Iterator[bytes] + """ + total_chunks = self._message_chunk_number(message) + message_id = random.randint(0, 0xFFFFFFFFFFFFFFFF) + for sequence, chunk in enumerate( + ( + message[i : i + self.chunk_size] + for i in range(0, len(message), self.chunk_size) + ) + ): + yield self._encode(message_id, sequence, total_chunks, chunk) + + def chunk_message(self, message): + """Chunk a GELF message + + Silently drop chunk overflowing GELF messages. + + :param message: GELF message to chunk. + :type message: bytes + + :return: Iterator of the chunks of a GELF message. + :rtype: Iterator[bytes], None + """ + if self._message_chunk_number(message) > GELF_MAX_CHUNK_NUMBER: + return + for chunk in self._gen_gelf_chunks(message): + yield chunk + + +class GELFChunkOverflowWarning(Warning): + """Warning that a chunked GELF UDP message requires more than 128 chunks""" + + +class GELFWarningChunker(BaseGELFChunker): + """GELF UDP message chunker that warns and drops overflowing messages""" + + def chunk_message(self, message): + """Chunk a GELF message + + Issue a :class:`.handler.GELFChunkOverflowWarning` on chunk + overflowing GELF messages. Then drop them. + """ + if self._message_chunk_number(message) > GELF_MAX_CHUNK_NUMBER: + warnings.warn( + "chunk overflowing GELF message: {}".format(message), + GELFChunkOverflowWarning, + ) + return + for chunk in self._gen_gelf_chunks(message): + yield chunk + + +class GELFTruncationFailureWarning(GELFChunkOverflowWarning): + """Warning that the truncation of a chunked GELF UDP message failed + to prevent chunk overflowing""" + + +class GELFTruncatingChunker(BaseGELFChunker): + """GELF UDP message chunker that truncates overflowing messages""" + + def __init__( + self, + chunk_size=WAN_CHUNK, + compress=True, + gelf_packer=BaseGELFHandler._pack_gelf_dict, + ): + """Initialize the GELFTruncatingChunker + + :param compress: Boolean noting whether the given GELF messages are + originally compressed. + :type compress: bool + + :param gelf_packer: Function handle for packing a GELF dictionary + into bytes. Should be of the form ``gelf_packer(gelf_dict)``. + :type gelf_packer: Callable[dict] + """ + BaseGELFChunker.__init__(self, chunk_size) + self.gelf_packer = gelf_packer + self.compress = compress + + def gen_chunk_overflow_gelf_log(self, raw_message): + """Attempt to truncate a chunk overflowing GELF message + + :param raw_message: Original bytes of a chunk overflowing GELF message. + :type raw_message: bytes + + :return: Truncated and simplified version of raw_message. + :rtype: bytes + """ + if self.compress: + message = zlib.decompress(raw_message) + else: + message = raw_message + + gelf_dict = json.loads(message.decode("UTF-8")) + # Simplified GELF message dictionary to base the truncated + # GELF message from + simplified_gelf_dict = { + "version": gelf_dict["version"], + "host": gelf_dict["host"], + "short_message": "", + "timestamp": gelf_dict["timestamp"], + "level": SYSLOG_LEVELS.get(logging.ERROR, logging.ERROR), + "facility": gelf_dict["facility"], + "_chunk_overflow": True, + } + + # compute a estimate of the number of message chunks left this is + # used to estimate the amount of truncation to apply + gelf_chunks_free = GELF_MAX_CHUNK_NUMBER - self._message_chunk_number( + zlib.compress(self.gelf_packer(simplified_gelf_dict)) + if self.compress + else self.gelf_packer(simplified_gelf_dict) + ) + truncated_short_message = gelf_dict["short_message"][ + : self.chunk_size * gelf_chunks_free + ] + for clip in range(gelf_chunks_free, -1, -1): + simplified_gelf_dict["short_message"] = truncated_short_message + packed_message = self.gelf_packer(simplified_gelf_dict) + if self.compress: + packed_message = zlib.compress(packed_message) + if self._message_chunk_number(packed_message) <= GELF_MAX_CHUNK_NUMBER: + return packed_message + else: + truncated_short_message = truncated_short_message[: -self.chunk_size] + else: + raise GELFTruncationFailureWarning( + "truncation failed preventing chunk overflowing for GELF message: {}".format( + raw_message + ) + ) + + def chunk_message(self, message): + """Chunk a GELF message + + Issue a :class:`.handler.GELFChunkOverflowWarning` on chunk + overflowing GELF messages. Then attempt to truncate and simplify the + chunk overflowing GELF message so that it may be successfully + chunked without overflowing. + + If the truncation and simplification of the chunk overflowing GELF + message fails issue a :class:`.handler.GELFTruncationFailureWarning` + and drop the overflowing GELF message. + """ + if self._message_chunk_number(message) > GELF_MAX_CHUNK_NUMBER: + warnings.warn( + "truncating GELF chunk overflowing message: {}".format(message), + GELFChunkOverflowWarning, + ) + try: + message = self.gen_chunk_overflow_gelf_log(message) + except GELFTruncationFailureWarning as w: + warnings.warn(w) + return + for chunk in self._gen_gelf_chunks(message): + yield chunk + + class GELFUDPHandler(BaseGELFHandler, DatagramHandler): - """Graylog Extended Log Format UDP handler""" + """GELF UDP handler""" - def __init__(self, host, port=12202, **kwargs): + def __init__(self, host, port=12202, gelf_chunker=GELFWarningChunker(), **kwargs): """Initialize the GELFUDPHandler - :param host: The host of the Graylog server. + .. note:: + By default a :class:`.handler.GELFWarningChunker` is used as the + ``gelf_chunker``. Thus, GELF messages that chunk overflow will + issue a :class:`.handler.GELFChunkOverflowWarning` and will be + dropped. + + :param host: GELF UDP input host. :type host: str - :param port: The port of the Graylog server (default ``12202``). + :param port: GELF UDP input port. :type port: int + + :param gelf_chunker: :class:`.handler.BaseGELFChunker` instance to + handle chunking larger GELF messages. + :type gelf_chunker: GELFWarningChunker """ BaseGELFHandler.__init__(self, **kwargs) DatagramHandler.__init__(self, host, port) + self.gelf_chunker = gelf_chunker def send(self, s): - if len(s) < self.chunk_size: - DatagramHandler.send(self, s) + if len(s) < self.gelf_chunker.chunk_size: + super(GELFUDPHandler, self).send(s) else: - for chunk in ChunkedGELF(s, self.chunk_size): - DatagramHandler.send(self, chunk) + for chunk in self.gelf_chunker.chunk_message(s): + super(GELFUDPHandler, self).send(chunk) class GELFTCPHandler(BaseGELFHandler, SocketHandler): - """Graylog Extended Log Format TCP handler""" + """GELF TCP handler""" def __init__(self, host, port=12201, **kwargs): """Initialize the GELFTCPHandler - :param host: The host of the Graylog server. + :param host: GELF TCP input host. :type host: str - :param port: The port of the Graylog server (default ``12201``). + :param port: GELF TCP input port. :type port: int + + .. attention:: + GELF TCP does not support compression due to the use of the null + byte (``\\0``) as frame delimiter. + + Thus, :class:`.handler.GELFTCPHandler` does not support setting + ``compress`` to :obj:`True` and is locked to :obj:`False`. """ BaseGELFHandler.__init__(self, compress=False, **kwargs) SocketHandler.__init__(self, host, port) def makePickle(self, record): - """Add a null terminator to a GELFTCPHandler's pickles as a TCP frame - object needs to be null terminated + """Add a null terminator to generated pickles as TCP frame objects + need to be null terminated :param record: :class:`logging.LogRecord` to create a null terminated GELF log. :type record: logging.LogRecord - :return: A GELF log encoded as a null terminated bytes string. + :return: Null terminated bytes representing a GELF log. :rtype: bytes """ - return BaseGELFHandler.makePickle(self, record) + b'\x00' + return super(GELFTCPHandler, self).makePickle(record) + b"\x00" class GELFTLSHandler(GELFTCPHandler): - """Graylog Extended Log Format TCP handler with TLS support""" + """GELF TCP handler with TLS support""" - def __init__(self, host, port=12204, validate=False, ca_certs=None, - certfile=None, keyfile=None, **kwargs): + def __init__( + self, + host, + port=12204, + validate=False, + ca_certs=None, + certfile=None, + keyfile=None, + **kwargs + ): """Initialize the GELFTLSHandler - :param host: The host of the Graylog server. + :param host: GELF TLS input host. :type host: str - :param port: The port of the Graylog server (default ``12204``). + :param port: GELF TLS input port. :type port: int - :param validate: If :obj:`True`, validate server certificate. - In that case specifying ``ca_certs`` is required. + :param validate: If :obj:`True`, validate the Graylog server's + certificate. In this case specifying ``ca_certs`` is also + required. :type validate: bool :param ca_certs: Path to CA bundle file. @@ -434,12 +686,11 @@ stored with the certificate, this parameter can be ignored. :type keyfile: str """ - if validate and ca_certs is None: - raise ValueError('CA bundle file path must be specified') + raise ValueError("CA bundle file path must be specified") if keyfile is not None and certfile is None: - raise ValueError('certfile must be specified') + raise ValueError("certfile must be specified") GELFTCPHandler.__init__(self, host=host, port=port, **kwargs) @@ -449,11 +700,10 @@ self.keyfile = keyfile if keyfile else certfile def makeSocket(self, timeout=1): - """Override SocketHandler.makeSocket, to allow creating wrapped - TLS sockets""" + """Create a TLS wrapped socket""" plain_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if hasattr(plain_socket, 'settimeout'): + if hasattr(plain_socket, "settimeout"): plain_socket.settimeout(timeout) wrapped_socket = ssl.wrap_socket( @@ -461,7 +711,7 @@ ca_certs=self.ca_certs, cert_reqs=self.reqs, keyfile=self.keyfile, - certfile=self.certfile + certfile=self.certfile, ) wrapped_socket.connect((self.host, self.port)) @@ -470,10 +720,11 @@ # TODO: add https? class GELFHTTPHandler(BaseGELFHandler): - """Graylog Extended Log Format HTTP handler""" + """GELF HTTP handler""" - def __init__(self, host, port=12203, compress=True, path='/gelf', - timeout=5, **kwargs): + def __init__( + self, host, port=12203, compress=True, path="/gelf", timeout=5, **kwargs + ): """Initialize the GELFHTTPHandler :param host: GELF HTTP input host. @@ -483,18 +734,17 @@ :type port: int :param compress: If :obj:`True` compress the GELF message before - sending it to the server (:obj:`True` by default). + sending it to the Graylog server. :type compress: bool :param path: Path of the HTTP input. (see http://docs.graylog.org/en/latest/pages/sending_data.html#gelf-via-http) :type path: str - :param timeout: Amount of seconds that HTTP client should wait before - it discards the request if the server doesn't respond. + :param timeout: Number of seconds the HTTP client should wait before + it discards the request if the Graylog server doesn't respond. :type timeout: int """ - BaseGELFHandler.__init__(self, compress=compress, **kwargs) self.host = host @@ -504,56 +754,18 @@ self.headers = {} if compress: - self.headers['Content-Encoding'] = 'gzip,deflate' + self.headers["Content-Encoding"] = "gzip,deflate" def emit(self, record): """Convert a :class:`logging.LogRecord` to GELF and emit it to Graylog - via an HTTP POST request + via a HTTP POST request - :param record: :class:`logging.LogRecord` to convert into a - Graylog GELF log and emit to Graylog via HTTP POST. + :param record: :class:`logging.LogRecord` to convert into a GELF log + and emit to Graylog via a HTTP POST request. :type record: logging.LogRecord """ pickle = self.makePickle(record) connection = httplib.HTTPConnection( - host=self.host, - port=self.port, - timeout=self.timeout + host=self.host, port=self.port, timeout=self.timeout ) - connection.request('POST', self.path, pickle, self.headers) - - -class ChunkedGELF(object): - """Class that chunks a message into a GLEF compatible chunks""" - - def __init__(self, message, size): - """Initialize the ChunkedGELF message class - - :param message: The message to chunk. - :type message: bytes - - :param size: The size of the chunks. - :type size: int - """ - self.message = message - self.size = size - self.pieces = \ - struct.pack('B', int(math.ceil(len(message) * 1.0 / size))) - self.id = struct.pack('Q', random.randint(0, 0xFFFFFFFFFFFFFFFF)) - - def message_chunks(self): - return (self.message[i:i + self.size] for i - in range(0, len(self.message), self.size)) - - def encode(self, sequence, chunk): - return b''.join([ - b'\x1e\x0f', - self.id, - struct.pack('B', sequence), - self.pieces, - chunk - ]) - - def __iter__(self): - for sequence, chunk in enumerate(self.message_chunks()): - yield self.encode(sequence, chunk) + connection.request("POST", self.path, pickle, self.headers) diff -Nru graypy-1.1.3/graypy/__init__.py graypy-2.1.0/graypy/__init__.py --- graypy-1.1.3/graypy/__init__.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/graypy/__init__.py 2019-09-30 22:34:35.000000000 +0000 @@ -3,16 +3,22 @@ """graypy -Python logging handler that sends messages in +Python logging handlers that send messages in the Graylog Extended Log Format (GELF). Modules: - + :mod:`.handler` - Logging Handlers that send messages in GELF - + :mod:`.rabbitmq` - RabbitMQ and BaseGELFHandler Logging Handler composition + + :mod:`.handler` - Basic GELF Logging Handlers + + :mod:`.rabbitmq` - RabbitMQ GELF Logging Handler """ -from graypy.handler import GELFUDPHandler, GELFTCPHandler, GELFTLSHandler, \ - GELFHTTPHandler, WAN_CHUNK, LAN_CHUNK +from graypy.handler import ( + GELFUDPHandler, + GELFTCPHandler, + GELFTLSHandler, + GELFHTTPHandler, + WAN_CHUNK, + LAN_CHUNK, +) try: from graypy.rabbitmq import GELFRabbitHandler, ExcludeFilter @@ -20,4 +26,4 @@ pass # amqplib is probably not installed -__version__ = (1, 1, 3) +__version__ = (2, 1, 0) diff -Nru graypy-1.1.3/graypy/rabbitmq.py graypy-2.1.0/graypy/rabbitmq.py --- graypy-1.1.3/graypy/rabbitmq.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/graypy/rabbitmq.py 2019-09-30 22:34:35.000000000 +0000 @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""Logging Handler integrating RabbitMQ and Graylog Extended Log Format (GELF) -handler""" +"""Logging Handler integrating RabbitMQ and +Graylog Extended Log Format (GELF)""" import json from logging import Filter @@ -23,26 +23,32 @@ class GELFRabbitHandler(BaseGELFHandler, SocketHandler): - """RabbitMQ / Graylog Extended Log Format handler + """RabbitMQ / GELF handler .. note:: This handler ignores all messages logged by amqplib. """ - def __init__(self, url, exchange='logging.gelf', exchange_type='fanout', - virtual_host='/', routing_key='', **kwargs): + def __init__( + self, + url, + exchange="logging.gelf", + exchange_type="fanout", + virtual_host="/", + routing_key="", + **kwargs + ): """Initialize the GELFRabbitHandler :param url: RabbitMQ URL (ex: amqp://guest:guest@localhost:5672/) :type url: str - :param exchange: RabbitMQ exchange. (default 'logging.gelf'). - A queue binding must be defined on the server to prevent - log messages from being dropped. + :param exchange: RabbitMQ exchange. A queue binding must be defined + on the server to prevent GELF logs from being dropped. :type exchange: str - :param exchange_type: RabbitMQ exchange type (default 'fanout'). + :param exchange_type: RabbitMQ exchange type. :type exchange_type: str :param virtual_host: @@ -53,32 +59,31 @@ """ self.url = url parsed = urlparse(url) - if parsed.scheme != 'amqp': + if parsed.scheme != "amqp": raise ValueError('invalid URL scheme (expected "amqp"): %s' % url) - host = parsed.hostname or 'localhost' + host = parsed.hostname or "localhost" port = _ifnone(parsed.port, 5672) - self.virtual_host = virtual_host if not unquote( - parsed.path[1:]) else unquote(parsed.path[1:]) + self.virtual_host = ( + virtual_host if not unquote(parsed.path[1:]) else unquote(parsed.path[1:]) + ) self.cn_args = { - 'host': '%s:%s' % (host, port), - 'userid': _ifnone(parsed.username, 'guest'), - 'password': _ifnone(parsed.password, 'guest'), - 'virtual_host': self.virtual_host, - 'insist': False, + "host": "%s:%s" % (host, port), + "userid": _ifnone(parsed.username, "guest"), + "password": _ifnone(parsed.password, "guest"), + "virtual_host": self.virtual_host, + "insist": False, } self.exchange = exchange self.exchange_type = exchange_type self.routing_key = routing_key - BaseGELFHandler.__init__( - self, - **kwargs - ) + BaseGELFHandler.__init__(self, **kwargs) SocketHandler.__init__(self, host, port) - self.addFilter(ExcludeFilter('amqplib')) + self.addFilter(ExcludeFilter("amqplib")) def makeSocket(self, timeout=1): - return RabbitSocket(self.cn_args, timeout, self.exchange, - self.exchange_type, self.routing_key) + return RabbitSocket( + self.cn_args, timeout, self.exchange, self.exchange_type, self.routing_key + ) def makePickle(self, record): message_dict = self._make_gelf_dict(record) @@ -92,8 +97,7 @@ self.exchange = exchange self.exchange_type = exchange_type self.routing_key = routing_key - self.connection = amqp.Connection( - connection_timeout=timeout, **self.cn_args) + self.connection = amqp.Connection(connection_timeout=timeout, **self.cn_args) self.channel = self.connection.channel() self.channel.exchange_declare( exchange=self.exchange, @@ -105,9 +109,7 @@ def sendall(self, data): msg = amqp.Message(data, delivery_mode=2) self.channel.basic_publish( - msg, - exchange=self.exchange, - routing_key=self.routing_key + msg, exchange=self.exchange, routing_key=self.routing_key ) def close(self): @@ -126,14 +128,16 @@ def __init__(self, name): """Initialize the ExcludeFilter - :param name: Name to match for within a:class:`logging.LogRecord`'s + :param name: Name to match for within a :class:`logging.LogRecord`'s ``name`` field for filtering. :type name: str """ if not name: - raise ValueError('ExcludeFilter requires a non-empty name') + raise ValueError("ExcludeFilter requires a non-empty name") Filter.__init__(self, name) def filter(self, record): - return not (record.name.startswith(self.name) and ( - len(record.name) == self.nlen or record.name[self.nlen] == ".")) + return not ( + record.name.startswith(self.name) + and (len(record.name) == self.nlen or record.name[self.nlen] == ".") + ) diff -Nru graypy-1.1.3/graypy.egg-info/PKG-INFO graypy-2.1.0/graypy.egg-info/PKG-INFO --- graypy-1.1.3/graypy.egg-info/PKG-INFO 2019-07-15 23:27:17.000000000 +0000 +++ graypy-2.1.0/graypy.egg-info/PKG-INFO 2019-09-30 22:39:23.000000000 +0000 @@ -1,20 +1,38 @@ Metadata-Version: 2.1 Name: graypy -Version: 1.1.3 -Summary: Python logging handler that sends messages in Graylog Extended Log Format (GLEF). +Version: 2.1.0 +Summary: Python logging handlers that send messages in the Graylog Extended Log Format (GELF). Home-page: https://github.com/severb/graypy Author: Sever Banesiu Author-email: banesiu.sever@gmail.com License: BSD License -Description: |PyPI_Status| - |Build_Status| - |Coverage_Status| +Description: ###### + graypy + ###### + + .. image:: https://img.shields.io/pypi/v/graypy.svg + :target: https://pypi.python.org/pypi/graypy + :alt: PyPI Status + + .. image:: https://travis-ci.org/severb/graypy.svg?branch=master + :target: https://travis-ci.org/severb/graypy + :alt: Build Status + + .. image:: https://readthedocs.org/projects/graypy/badge/?version=stable + :target: https://graypy.readthedocs.io/en/stable/?badge=stable + :alt: Documentation Status + + .. image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/severb/graypy + :alt: Coverage Status Description =========== - Python logging handlers that send messages in the Graylog Extended - Log Format (GELF_). + Python logging handlers that send log messages in the + Graylog Extended Log Format (GELF_). + + graypy supports sending GELF logs to both Graylog2 and Graylog3 servers. Installing ========== @@ -22,38 +40,52 @@ Using pip --------- - Install the basic graypy python logging handlers + Install the basic graypy python logging handlers: - .. code-block:: bash + .. code-block:: console pip install graypy - Install with requirements for ``GELFRabbitHandler`` + Install with requirements for ``GELFRabbitHandler``: - .. code-block:: bash + .. code-block:: console pip install graypy[amqp] Using easy_install ------------------ - Install the basic graypy python logging handlers + Install the basic graypy python logging handlers: - .. code-block:: bash + .. code-block:: console - easy_install graypy + easy_install graypy - Install with requirements for ``GELFRabbitHandler`` + Install with requirements for ``GELFRabbitHandler``: - .. code-block:: bash + .. code-block:: console - easy_install graypy[amqp] + easy_install graypy[amqp] Usage ===== - Messages are sent to Graylog2 using a custom handler for the builtin logging - library in GELF format + graypy sends GELF logs to a Graylog server via subclasses of the python + `logging.Handler`_ class. + + Below is the list of ready to run GELF logging handlers defined by graypy: + + * ``GELFUDPHandler`` - UDP log forwarding + * ``GELFTCPHandler`` - TCP log forwarding + * ``GELFTLSHandler`` - TCP log forwarding with TLS support + * ``GELFHTTPHandler`` - HTTP log forwarding + * ``GELFRabbitHandler`` - RabbitMQ log forwarding + + UDP Logging + ----------- + + UDP Log forwarding to a locally hosted Graylog server can be easily done with + the ``GELFUDPHandler``: .. code-block:: python @@ -66,30 +98,36 @@ handler = graypy.GELFUDPHandler('localhost', 12201) my_logger.addHandler(handler) - my_logger.debug('Hello Graylog2.') + my_logger.debug('Hello Graylog.') - Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and - configure your Graylog2 server to consume messages via AMQP. This prevents - log messages from being lost due to dropped UDP packets (``GELFUDPHandler`` - sends messages to Graylog2 using UDP). You will need to configure RabbitMQ - with a 'gelf_log' queue and bind it to the 'logging.gelf' exchange so - messages are properly routed to a queue that can be consumed by - Graylog2 (the queue and exchange names may be customized to your liking) - .. code-block:: python + UDP GELF Chunkers + ^^^^^^^^^^^^^^^^^ - import logging - import graypy + `GELF UDP Chunking`_ is supported by the ``GELFUDPHandler`` and is defined by + the ``gelf_chunker`` argument within its constructor. By default the + ``GELFWarningChunker`` is used, thus, GELF messages that chunk overflow + (i.e. consisting of more than 128 chunks) will issue a + ``GELFChunkOverflowWarning`` and **will be dropped**. - my_logger = logging.getLogger('test_logger') - my_logger.setLevel(logging.DEBUG) + Other ``gelf_chunker`` options are also available: - handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') - my_logger.addHandler(handler) + * ``BaseGELFChunker`` silently drops GELF messages that chunk overflow + * ``GELFTruncatingChunker`` issues a ``GELFChunkOverflowWarning`` and + simplifies and truncates GELF messages that chunk overflow in a attempt + to send some content to Graylog. If this process fails to prevent + another chunk overflow a ``GELFTruncationFailureWarning`` is issued. - my_logger.debug('Hello Graylog2.') + RabbitMQ Logging + ---------------- - Tracebacks are added as full messages + Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and + configure your Graylog server to consume messages via AMQP. This prevents log + messages from being lost due to dropped UDP packets (``GELFUDPHandler`` sends + messages to Graylog using UDP). You will need to configure RabbitMQ with a + ``gelf_log`` queue and bind it to the ``logging.gelf`` exchange so messages + are properly routed to a queue that can be consumed by Graylog (the queue and + exchange names may be customized to your liking). .. code-block:: python @@ -99,20 +137,13 @@ my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) - handler = graypy.GELFUDPHandler('localhost', 12201) + handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') my_logger.addHandler(handler) - try: - puff_the_magic_dragon() - except NameError: - my_logger.debug('No dragons here.', exc_info=1) + my_logger.debug('Hello Graylog.') - - For more detailed usage information please see the documentation provided - within graypy's handler's docstrings. - - Using with Django - ================= + Django Logging + -------------- It's easy to integrate ``graypy`` with Django's logging settings. Just add a new handler in your ``settings.py``: @@ -120,8 +151,8 @@ .. code-block:: python LOGGING = { - ... - + 'version': 1, + # other dictConfig keys here... 'handlers': { 'graypy': { 'level': 'WARNING', @@ -130,7 +161,6 @@ 'port': 12201, }, }, - 'loggers': { 'django.request': { 'handlers': ['graypy'], @@ -140,29 +170,57 @@ }, } - Custom fields - ============= + Traceback Logging + ----------------- + + By default log captured exception tracebacks are added to the GELF log as + ``full_message`` fields: + + .. code-block:: python + + import logging + import graypy + + my_logger = logging.getLogger('test_logger') + my_logger.setLevel(logging.DEBUG) + + handler = graypy.GELFUDPHandler('localhost', 12201) + my_logger.addHandler(handler) + + try: + puff_the_magic_dragon() + except NameError: + my_logger.debug('No dragons here.', exc_info=1) + + Default Logging Fields + ---------------------- + + By default a number of debugging logging fields are automatically added to the + GELF log if available: - A number of custom fields are automatically added if available: * function * pid * process_name * thread_name - You can disable these additional fields if you don't want them by adding - an the ``debugging_fields=False`` to the handler: + You can disable automatically adding these debugging logging fields by + specifying ``debugging_fields=False`` in the handler's constructor: .. code-block:: python handler = graypy.GELFUDPHandler('localhost', 12201, debugging_fields=False) - graypy also supports additional fields to be included in the messages sent - to Graylog2. This can be done by using Python's LoggerAdapter_ and - Filter_. In general, LoggerAdapter makes it easy to add static information - to your log messages and Filters give you more flexibility, for example to - add additional information based on the message that is being logged. + Adding Custom Logging Fields + ---------------------------- + + graypy also supports including custom fields in the GELF logs sent to Graylog. + This can be done by using Python's LoggerAdapter_ and Filter_ classes. + + Using LoggerAdapter + ^^^^^^^^^^^^^^^^^^^ - Example using LoggerAdapter_ + LoggerAdapter_ makes it easy to add static information to your GELF log + messages: .. code-block:: python @@ -178,9 +236,13 @@ my_adapter = logging.LoggerAdapter(logging.getLogger('test_logger'), {'username': 'John'}) - my_adapter.debug('Hello Graylog2 from John.') + my_adapter.debug('Hello Graylog from John.') - Example using Filter_ + Using Filter + ^^^^^^^^^^^^ + + Filter_ gives more flexibility and allows for dynamic information to be + added to your GELF logs: .. code-block:: python @@ -191,7 +253,7 @@ def __init__(self): # In an actual use case would dynamically get this # (e.g. from memcache) - self.username = "John" + self.username = 'John' def filter(self, record): record.username = self.username @@ -205,28 +267,22 @@ my_logger.addFilter(UsernameFilter()) - my_logger.debug('Hello Graylog2 from John.') + my_logger.debug('Hello Graylog from John.') - Contributors: + Contributors + ============ * Sever Banesiu * Daniel Miller * Tushar Makkar * Nathan Klapstein - .. _GELF: http://docs.graylog.org/en/latest/pages/gelf.html - .. _LoggerAdapter: http://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information - .. _Filter: http://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information - - .. |Build_Status| image:: https://travis-ci.org/severb/graypy.svg?branch=master - :target: https://travis-ci.org/severb/graypy - + .. _GELF: https://docs.graylog.org/en/latest/pages/gelf.html + .. _logging.Handler: https://docs.python.org/3/library/logging.html#logging.Handler + .. _GELF UDP Chunking: https://docs.graylog.org/en/latest/pages/gelf.html#chunking + .. _LoggerAdapter: https://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information + .. _Filter: https://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information - .. |Coverage_Status| image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg - :target: https://codecov.io/gh/severb/graypy - - .. |PyPI_Status| image:: https://img.shields.io/pypi/v/graypy.svg - :target: https://pypi.python.org/pypi/graypy Keywords: logging gelf graylog2 graylog udp amqp Platform: UNKNOWN Classifier: License :: OSI Approved :: BSD License @@ -245,4 +301,5 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: System :: Logging Description-Content-Type: text/x-rst +Provides-Extra: docs Provides-Extra: amqp diff -Nru graypy-1.1.3/graypy.egg-info/requires.txt graypy-2.1.0/graypy.egg-info/requires.txt --- graypy-1.1.3/graypy.egg-info/requires.txt 2019-07-15 23:27:17.000000000 +0000 +++ graypy-2.1.0/graypy.egg-info/requires.txt 2019-09-30 22:39:23.000000000 +0000 @@ -1,3 +1,8 @@ [amqp] amqplib==1.0.2 + +[docs] +sphinx<3.0.0,>=2.1.2 +sphinx_rtd_theme<1.0.0,>=0.4.3 +sphinx-autodoc-typehints<2.0.0,>=1.6.0 diff -Nru graypy-1.1.3/graypy.egg-info/SOURCES.txt graypy-2.1.0/graypy.egg-info/SOURCES.txt --- graypy-1.1.3/graypy.egg-info/SOURCES.txt 2019-07-15 23:27:17.000000000 +0000 +++ graypy-2.1.0/graypy.egg-info/SOURCES.txt 2019-09-30 22:39:23.000000000 +0000 @@ -14,6 +14,11 @@ graypy.egg-info/top_level.txt tests/__init__.py tests/helper.py +tests/config/create_ssl_certs.sh +tests/config/docker-compose.yml +tests/config/inputs.json +tests/config/start_local_graylog_server.sh +tests/config/stop_local_graylog_server.sh tests/integration/__init__.py tests/integration/helper.py tests/integration/test_chunked_logging.py @@ -25,4 +30,5 @@ tests/unit/helper.py tests/unit/test_ExcludeFilter.py tests/unit/test_GELFRabbitHandler.py +tests/unit/test_chunking.py tests/unit/test_handler.py \ No newline at end of file diff -Nru graypy-1.1.3/MANIFEST.in graypy-2.1.0/MANIFEST.in --- graypy-1.1.3/MANIFEST.in 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/MANIFEST.in 2019-09-30 22:34:35.000000000 +0000 @@ -1,3 +1,4 @@ include LICENSE include README.rst -recursive-include tests *.py \ No newline at end of file +recursive-include tests *.py +recursive-include tests/config * \ No newline at end of file diff -Nru graypy-1.1.3/PKG-INFO graypy-2.1.0/PKG-INFO --- graypy-1.1.3/PKG-INFO 2019-07-15 23:27:17.000000000 +0000 +++ graypy-2.1.0/PKG-INFO 2019-09-30 22:39:23.000000000 +0000 @@ -1,20 +1,38 @@ Metadata-Version: 2.1 Name: graypy -Version: 1.1.3 -Summary: Python logging handler that sends messages in Graylog Extended Log Format (GLEF). +Version: 2.1.0 +Summary: Python logging handlers that send messages in the Graylog Extended Log Format (GELF). Home-page: https://github.com/severb/graypy Author: Sever Banesiu Author-email: banesiu.sever@gmail.com License: BSD License -Description: |PyPI_Status| - |Build_Status| - |Coverage_Status| +Description: ###### + graypy + ###### + + .. image:: https://img.shields.io/pypi/v/graypy.svg + :target: https://pypi.python.org/pypi/graypy + :alt: PyPI Status + + .. image:: https://travis-ci.org/severb/graypy.svg?branch=master + :target: https://travis-ci.org/severb/graypy + :alt: Build Status + + .. image:: https://readthedocs.org/projects/graypy/badge/?version=stable + :target: https://graypy.readthedocs.io/en/stable/?badge=stable + :alt: Documentation Status + + .. image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/severb/graypy + :alt: Coverage Status Description =========== - Python logging handlers that send messages in the Graylog Extended - Log Format (GELF_). + Python logging handlers that send log messages in the + Graylog Extended Log Format (GELF_). + + graypy supports sending GELF logs to both Graylog2 and Graylog3 servers. Installing ========== @@ -22,38 +40,52 @@ Using pip --------- - Install the basic graypy python logging handlers + Install the basic graypy python logging handlers: - .. code-block:: bash + .. code-block:: console pip install graypy - Install with requirements for ``GELFRabbitHandler`` + Install with requirements for ``GELFRabbitHandler``: - .. code-block:: bash + .. code-block:: console pip install graypy[amqp] Using easy_install ------------------ - Install the basic graypy python logging handlers + Install the basic graypy python logging handlers: - .. code-block:: bash + .. code-block:: console - easy_install graypy + easy_install graypy - Install with requirements for ``GELFRabbitHandler`` + Install with requirements for ``GELFRabbitHandler``: - .. code-block:: bash + .. code-block:: console - easy_install graypy[amqp] + easy_install graypy[amqp] Usage ===== - Messages are sent to Graylog2 using a custom handler for the builtin logging - library in GELF format + graypy sends GELF logs to a Graylog server via subclasses of the python + `logging.Handler`_ class. + + Below is the list of ready to run GELF logging handlers defined by graypy: + + * ``GELFUDPHandler`` - UDP log forwarding + * ``GELFTCPHandler`` - TCP log forwarding + * ``GELFTLSHandler`` - TCP log forwarding with TLS support + * ``GELFHTTPHandler`` - HTTP log forwarding + * ``GELFRabbitHandler`` - RabbitMQ log forwarding + + UDP Logging + ----------- + + UDP Log forwarding to a locally hosted Graylog server can be easily done with + the ``GELFUDPHandler``: .. code-block:: python @@ -66,30 +98,36 @@ handler = graypy.GELFUDPHandler('localhost', 12201) my_logger.addHandler(handler) - my_logger.debug('Hello Graylog2.') + my_logger.debug('Hello Graylog.') - Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and - configure your Graylog2 server to consume messages via AMQP. This prevents - log messages from being lost due to dropped UDP packets (``GELFUDPHandler`` - sends messages to Graylog2 using UDP). You will need to configure RabbitMQ - with a 'gelf_log' queue and bind it to the 'logging.gelf' exchange so - messages are properly routed to a queue that can be consumed by - Graylog2 (the queue and exchange names may be customized to your liking) - .. code-block:: python + UDP GELF Chunkers + ^^^^^^^^^^^^^^^^^ - import logging - import graypy + `GELF UDP Chunking`_ is supported by the ``GELFUDPHandler`` and is defined by + the ``gelf_chunker`` argument within its constructor. By default the + ``GELFWarningChunker`` is used, thus, GELF messages that chunk overflow + (i.e. consisting of more than 128 chunks) will issue a + ``GELFChunkOverflowWarning`` and **will be dropped**. - my_logger = logging.getLogger('test_logger') - my_logger.setLevel(logging.DEBUG) + Other ``gelf_chunker`` options are also available: - handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') - my_logger.addHandler(handler) + * ``BaseGELFChunker`` silently drops GELF messages that chunk overflow + * ``GELFTruncatingChunker`` issues a ``GELFChunkOverflowWarning`` and + simplifies and truncates GELF messages that chunk overflow in a attempt + to send some content to Graylog. If this process fails to prevent + another chunk overflow a ``GELFTruncationFailureWarning`` is issued. - my_logger.debug('Hello Graylog2.') + RabbitMQ Logging + ---------------- - Tracebacks are added as full messages + Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and + configure your Graylog server to consume messages via AMQP. This prevents log + messages from being lost due to dropped UDP packets (``GELFUDPHandler`` sends + messages to Graylog using UDP). You will need to configure RabbitMQ with a + ``gelf_log`` queue and bind it to the ``logging.gelf`` exchange so messages + are properly routed to a queue that can be consumed by Graylog (the queue and + exchange names may be customized to your liking). .. code-block:: python @@ -99,20 +137,13 @@ my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) - handler = graypy.GELFUDPHandler('localhost', 12201) + handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') my_logger.addHandler(handler) - try: - puff_the_magic_dragon() - except NameError: - my_logger.debug('No dragons here.', exc_info=1) + my_logger.debug('Hello Graylog.') - - For more detailed usage information please see the documentation provided - within graypy's handler's docstrings. - - Using with Django - ================= + Django Logging + -------------- It's easy to integrate ``graypy`` with Django's logging settings. Just add a new handler in your ``settings.py``: @@ -120,8 +151,8 @@ .. code-block:: python LOGGING = { - ... - + 'version': 1, + # other dictConfig keys here... 'handlers': { 'graypy': { 'level': 'WARNING', @@ -130,7 +161,6 @@ 'port': 12201, }, }, - 'loggers': { 'django.request': { 'handlers': ['graypy'], @@ -140,29 +170,57 @@ }, } - Custom fields - ============= + Traceback Logging + ----------------- + + By default log captured exception tracebacks are added to the GELF log as + ``full_message`` fields: + + .. code-block:: python + + import logging + import graypy + + my_logger = logging.getLogger('test_logger') + my_logger.setLevel(logging.DEBUG) + + handler = graypy.GELFUDPHandler('localhost', 12201) + my_logger.addHandler(handler) + + try: + puff_the_magic_dragon() + except NameError: + my_logger.debug('No dragons here.', exc_info=1) + + Default Logging Fields + ---------------------- + + By default a number of debugging logging fields are automatically added to the + GELF log if available: - A number of custom fields are automatically added if available: * function * pid * process_name * thread_name - You can disable these additional fields if you don't want them by adding - an the ``debugging_fields=False`` to the handler: + You can disable automatically adding these debugging logging fields by + specifying ``debugging_fields=False`` in the handler's constructor: .. code-block:: python handler = graypy.GELFUDPHandler('localhost', 12201, debugging_fields=False) - graypy also supports additional fields to be included in the messages sent - to Graylog2. This can be done by using Python's LoggerAdapter_ and - Filter_. In general, LoggerAdapter makes it easy to add static information - to your log messages and Filters give you more flexibility, for example to - add additional information based on the message that is being logged. + Adding Custom Logging Fields + ---------------------------- + + graypy also supports including custom fields in the GELF logs sent to Graylog. + This can be done by using Python's LoggerAdapter_ and Filter_ classes. + + Using LoggerAdapter + ^^^^^^^^^^^^^^^^^^^ - Example using LoggerAdapter_ + LoggerAdapter_ makes it easy to add static information to your GELF log + messages: .. code-block:: python @@ -178,9 +236,13 @@ my_adapter = logging.LoggerAdapter(logging.getLogger('test_logger'), {'username': 'John'}) - my_adapter.debug('Hello Graylog2 from John.') + my_adapter.debug('Hello Graylog from John.') - Example using Filter_ + Using Filter + ^^^^^^^^^^^^ + + Filter_ gives more flexibility and allows for dynamic information to be + added to your GELF logs: .. code-block:: python @@ -191,7 +253,7 @@ def __init__(self): # In an actual use case would dynamically get this # (e.g. from memcache) - self.username = "John" + self.username = 'John' def filter(self, record): record.username = self.username @@ -205,28 +267,22 @@ my_logger.addFilter(UsernameFilter()) - my_logger.debug('Hello Graylog2 from John.') + my_logger.debug('Hello Graylog from John.') - Contributors: + Contributors + ============ * Sever Banesiu * Daniel Miller * Tushar Makkar * Nathan Klapstein - .. _GELF: http://docs.graylog.org/en/latest/pages/gelf.html - .. _LoggerAdapter: http://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information - .. _Filter: http://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information - - .. |Build_Status| image:: https://travis-ci.org/severb/graypy.svg?branch=master - :target: https://travis-ci.org/severb/graypy - + .. _GELF: https://docs.graylog.org/en/latest/pages/gelf.html + .. _logging.Handler: https://docs.python.org/3/library/logging.html#logging.Handler + .. _GELF UDP Chunking: https://docs.graylog.org/en/latest/pages/gelf.html#chunking + .. _LoggerAdapter: https://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information + .. _Filter: https://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information - .. |Coverage_Status| image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg - :target: https://codecov.io/gh/severb/graypy - - .. |PyPI_Status| image:: https://img.shields.io/pypi/v/graypy.svg - :target: https://pypi.python.org/pypi/graypy Keywords: logging gelf graylog2 graylog udp amqp Platform: UNKNOWN Classifier: License :: OSI Approved :: BSD License @@ -245,4 +301,5 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: System :: Logging Description-Content-Type: text/x-rst +Provides-Extra: docs Provides-Extra: amqp diff -Nru graypy-1.1.3/README.rst graypy-2.1.0/README.rst --- graypy-1.1.3/README.rst 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/README.rst 2019-09-30 22:34:35.000000000 +0000 @@ -1,12 +1,30 @@ -|PyPI_Status| -|Build_Status| -|Coverage_Status| +###### +graypy +###### + +.. image:: https://img.shields.io/pypi/v/graypy.svg + :target: https://pypi.python.org/pypi/graypy + :alt: PyPI Status + +.. image:: https://travis-ci.org/severb/graypy.svg?branch=master + :target: https://travis-ci.org/severb/graypy + :alt: Build Status + +.. image:: https://readthedocs.org/projects/graypy/badge/?version=stable + :target: https://graypy.readthedocs.io/en/stable/?badge=stable + :alt: Documentation Status + +.. image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/severb/graypy + :alt: Coverage Status Description =========== -Python logging handlers that send messages in the Graylog Extended -Log Format (GELF_). +Python logging handlers that send log messages in the +Graylog Extended Log Format (GELF_). + +graypy supports sending GELF logs to both Graylog2 and Graylog3 servers. Installing ========== @@ -14,38 +32,52 @@ Using pip --------- -Install the basic graypy python logging handlers +Install the basic graypy python logging handlers: -.. code-block:: bash +.. code-block:: console pip install graypy -Install with requirements for ``GELFRabbitHandler`` +Install with requirements for ``GELFRabbitHandler``: -.. code-block:: bash +.. code-block:: console pip install graypy[amqp] Using easy_install ------------------ -Install the basic graypy python logging handlers +Install the basic graypy python logging handlers: -.. code-block:: bash +.. code-block:: console - easy_install graypy + easy_install graypy -Install with requirements for ``GELFRabbitHandler`` +Install with requirements for ``GELFRabbitHandler``: -.. code-block:: bash +.. code-block:: console - easy_install graypy[amqp] + easy_install graypy[amqp] Usage ===== -Messages are sent to Graylog2 using a custom handler for the builtin logging -library in GELF format +graypy sends GELF logs to a Graylog server via subclasses of the python +`logging.Handler`_ class. + +Below is the list of ready to run GELF logging handlers defined by graypy: + +* ``GELFUDPHandler`` - UDP log forwarding +* ``GELFTCPHandler`` - TCP log forwarding +* ``GELFTLSHandler`` - TCP log forwarding with TLS support +* ``GELFHTTPHandler`` - HTTP log forwarding +* ``GELFRabbitHandler`` - RabbitMQ log forwarding + +UDP Logging +----------- + +UDP Log forwarding to a locally hosted Graylog server can be easily done with +the ``GELFUDPHandler``: .. code-block:: python @@ -58,30 +90,36 @@ handler = graypy.GELFUDPHandler('localhost', 12201) my_logger.addHandler(handler) - my_logger.debug('Hello Graylog2.') + my_logger.debug('Hello Graylog.') -Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and -configure your Graylog2 server to consume messages via AMQP. This prevents -log messages from being lost due to dropped UDP packets (``GELFUDPHandler`` -sends messages to Graylog2 using UDP). You will need to configure RabbitMQ -with a 'gelf_log' queue and bind it to the 'logging.gelf' exchange so -messages are properly routed to a queue that can be consumed by -Graylog2 (the queue and exchange names may be customized to your liking) -.. code-block:: python +UDP GELF Chunkers +^^^^^^^^^^^^^^^^^ - import logging - import graypy +`GELF UDP Chunking`_ is supported by the ``GELFUDPHandler`` and is defined by +the ``gelf_chunker`` argument within its constructor. By default the +``GELFWarningChunker`` is used, thus, GELF messages that chunk overflow +(i.e. consisting of more than 128 chunks) will issue a +``GELFChunkOverflowWarning`` and **will be dropped**. - my_logger = logging.getLogger('test_logger') - my_logger.setLevel(logging.DEBUG) +Other ``gelf_chunker`` options are also available: - handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') - my_logger.addHandler(handler) +* ``BaseGELFChunker`` silently drops GELF messages that chunk overflow +* ``GELFTruncatingChunker`` issues a ``GELFChunkOverflowWarning`` and + simplifies and truncates GELF messages that chunk overflow in a attempt + to send some content to Graylog. If this process fails to prevent + another chunk overflow a ``GELFTruncationFailureWarning`` is issued. - my_logger.debug('Hello Graylog2.') +RabbitMQ Logging +---------------- -Tracebacks are added as full messages +Alternately, use ``GELFRabbitHandler`` to send messages to RabbitMQ and +configure your Graylog server to consume messages via AMQP. This prevents log +messages from being lost due to dropped UDP packets (``GELFUDPHandler`` sends +messages to Graylog using UDP). You will need to configure RabbitMQ with a +``gelf_log`` queue and bind it to the ``logging.gelf`` exchange so messages +are properly routed to a queue that can be consumed by Graylog (the queue and +exchange names may be customized to your liking). .. code-block:: python @@ -91,20 +129,13 @@ my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) - handler = graypy.GELFUDPHandler('localhost', 12201) + handler = graypy.GELFRabbitHandler('amqp://guest:guest@localhost/', exchange='logging.gelf') my_logger.addHandler(handler) - try: - puff_the_magic_dragon() - except NameError: - my_logger.debug('No dragons here.', exc_info=1) - + my_logger.debug('Hello Graylog.') -For more detailed usage information please see the documentation provided -within graypy's handler's docstrings. - -Using with Django -================= +Django Logging +-------------- It's easy to integrate ``graypy`` with Django's logging settings. Just add a new handler in your ``settings.py``: @@ -112,8 +143,8 @@ .. code-block:: python LOGGING = { - ... - + 'version': 1, + # other dictConfig keys here... 'handlers': { 'graypy': { 'level': 'WARNING', @@ -122,7 +153,6 @@ 'port': 12201, }, }, - 'loggers': { 'django.request': { 'handlers': ['graypy'], @@ -132,29 +162,57 @@ }, } -Custom fields -============= +Traceback Logging +----------------- + +By default log captured exception tracebacks are added to the GELF log as +``full_message`` fields: + +.. code-block:: python + + import logging + import graypy + + my_logger = logging.getLogger('test_logger') + my_logger.setLevel(logging.DEBUG) + + handler = graypy.GELFUDPHandler('localhost', 12201) + my_logger.addHandler(handler) + + try: + puff_the_magic_dragon() + except NameError: + my_logger.debug('No dragons here.', exc_info=1) + +Default Logging Fields +---------------------- + +By default a number of debugging logging fields are automatically added to the +GELF log if available: -A number of custom fields are automatically added if available: * function * pid * process_name * thread_name -You can disable these additional fields if you don't want them by adding -an the ``debugging_fields=False`` to the handler: +You can disable automatically adding these debugging logging fields by +specifying ``debugging_fields=False`` in the handler's constructor: .. code-block:: python handler = graypy.GELFUDPHandler('localhost', 12201, debugging_fields=False) -graypy also supports additional fields to be included in the messages sent -to Graylog2. This can be done by using Python's LoggerAdapter_ and -Filter_. In general, LoggerAdapter makes it easy to add static information -to your log messages and Filters give you more flexibility, for example to -add additional information based on the message that is being logged. +Adding Custom Logging Fields +---------------------------- + +graypy also supports including custom fields in the GELF logs sent to Graylog. +This can be done by using Python's LoggerAdapter_ and Filter_ classes. + +Using LoggerAdapter +^^^^^^^^^^^^^^^^^^^ -Example using LoggerAdapter_ +LoggerAdapter_ makes it easy to add static information to your GELF log +messages: .. code-block:: python @@ -170,9 +228,13 @@ my_adapter = logging.LoggerAdapter(logging.getLogger('test_logger'), {'username': 'John'}) - my_adapter.debug('Hello Graylog2 from John.') + my_adapter.debug('Hello Graylog from John.') -Example using Filter_ +Using Filter +^^^^^^^^^^^^ + +Filter_ gives more flexibility and allows for dynamic information to be +added to your GELF logs: .. code-block:: python @@ -183,7 +245,7 @@ def __init__(self): # In an actual use case would dynamically get this # (e.g. from memcache) - self.username = "John" + self.username = 'John' def filter(self, record): record.username = self.username @@ -197,25 +259,18 @@ my_logger.addFilter(UsernameFilter()) - my_logger.debug('Hello Graylog2 from John.') + my_logger.debug('Hello Graylog from John.') -Contributors: +Contributors +============ * Sever Banesiu * Daniel Miller * Tushar Makkar * Nathan Klapstein -.. _GELF: http://docs.graylog.org/en/latest/pages/gelf.html -.. _LoggerAdapter: http://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information -.. _Filter: http://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information - -.. |Build_Status| image:: https://travis-ci.org/severb/graypy.svg?branch=master - :target: https://travis-ci.org/severb/graypy - - -.. |Coverage_Status| image:: https://codecov.io/gh/severb/graypy/branch/master/graph/badge.svg - :target: https://codecov.io/gh/severb/graypy - -.. |PyPI_Status| image:: https://img.shields.io/pypi/v/graypy.svg - :target: https://pypi.python.org/pypi/graypy \ No newline at end of file +.. _GELF: https://docs.graylog.org/en/latest/pages/gelf.html +.. _logging.Handler: https://docs.python.org/3/library/logging.html#logging.Handler +.. _GELF UDP Chunking: https://docs.graylog.org/en/latest/pages/gelf.html#chunking +.. _LoggerAdapter: https://docs.python.org/howto/logging-cookbook.html#using-loggeradapters-to-impart-contextual-information +.. _Filter: https://docs.python.org/howto/logging-cookbook.html#using-filters-to-impart-contextual-information diff -Nru graypy-1.1.3/setup.cfg graypy-2.1.0/setup.cfg --- graypy-1.1.3/setup.cfg 2019-07-15 23:27:17.000000000 +0000 +++ graypy-2.1.0/setup.cfg 2019-09-30 22:39:23.000000000 +0000 @@ -1,6 +1,9 @@ [bdist_wheel] universal = 1 +[metadata] +license_file = LICENSE + [egg_info] tag_build = tag_date = 0 diff -Nru graypy-1.1.3/setup.py graypy-2.1.0/setup.py --- graypy-1.1.3/setup.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/setup.py 2019-09-30 22:34:35.000000000 +0000 @@ -13,7 +13,9 @@ def find_version(*file_paths): - with codecs.open(os.path.join(os.path.abspath(os.path.dirname(__file__)), *file_paths), 'r') as fp: + with codecs.open( + os.path.join(os.path.abspath(os.path.dirname(__file__)), *file_paths), "r" + ) as fp: version_file = fp.read() m = re.search(r"^__version__ = \((\d+), ?(\d+), ?(\d+)\)", version_file, re.M) if m: @@ -27,12 +29,22 @@ class Pylint(test): def run_tests(self): from pylint.lint import Run - Run(["graypy", "--persistent", "y", "--rcfile", ".pylintrc", - "--output-format", "colorized"]) + + Run( + [ + "graypy", + "--persistent", + "y", + "--rcfile", + ".pylintrc", + "--output-format", + "colorized", + ] + ) class PyTest(test): - user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] + user_options = [("pytest-args=", "a", "Arguments to pass to pytest")] def initialize_options(self): test.initialize_options(self) @@ -40,23 +52,25 @@ def run_tests(self): import shlex + # import here, cause outside the eggs aren't loaded import pytest + errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) setup( - name='graypy', + name="graypy", version=VERSION, - description="Python logging handler that sends messages in Graylog Extended Log Format (GLEF).", - long_description=open('README.rst').read(), + description="Python logging handlers that send messages in the Graylog Extended Log Format (GELF).", + long_description=open("README.rst").read(), long_description_content_type="text/x-rst", - keywords='logging gelf graylog2 graylog udp amqp', - author='Sever Banesiu', - author_email='banesiu.sever@gmail.com', - url='https://github.com/severb/graypy', - license='BSD License', + keywords="logging gelf graylog2 graylog udp amqp", + author="Sever Banesiu", + author_email="banesiu.sever@gmail.com", + url="https://github.com/severb/graypy", + license="BSD License", packages=find_packages(), include_package_data=True, zip_safe=False, @@ -66,26 +80,32 @@ "pylint>=1.9.3,<2.0.0", "mock>=2.0.0,<3.0.0", "requests>=2.20.1,<3.0.0", - "amqplib>=1.0.2,<2.0.0" + "amqplib>=1.0.2,<2.0.0", ], - extras_require={'amqp': ['amqplib==1.0.2']}, + extras_require={ + "amqp": ["amqplib==1.0.2"], + "docs": [ + "sphinx>=2.1.2,<3.0.0", + "sphinx_rtd_theme>=0.4.3,<1.0.0", + "sphinx-autodoc-typehints>=1.6.0,<2.0.0", + ], + }, classifiers=[ - 'License :: OSI Approved :: BSD License', - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: System :: Logging', + "License :: OSI Approved :: BSD License", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: System :: Logging", ], cmdclass={"test": PyTest, "lint": Pylint}, ) - diff -Nru graypy-1.1.3/tests/config/create_ssl_certs.sh graypy-2.1.0/tests/config/create_ssl_certs.sh --- graypy-1.1.3/tests/config/create_ssl_certs.sh 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/config/create_ssl_certs.sh 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,167 @@ +#!/bin/bash -e + +# +# This scrip generate self-signed certificate to be used in development. +# it sets the CN to the first provided hostname and will add all other +# provided names to subjectAltName. +# +# Some Magic is added to the script that tries to find some settings for the +# current host where this script is started. +# +# This script was first created by Jan Doberstein 2017-07-30 +# +# This script is tested on CentOS 7, Ubuntu 14.04, Ubuntu 16.04, MacOS 10.12 + +OPSSLBIN=$(which openssl) + +while getopts "d:h:i:m?" opt; do + case ${opt} in + h) HNAME+=("${OPTARG}");; + i) HIP+=("${OPTARG}");; + m) MMODE=active;; + d) VALIDDAYS=${OPTARG};; + s) KEYSECRET=${OPTARG};; + ?) HELPME=yes;; + *) HELPME=yes;; + esac +done + +if [ -n "${HELPME}" ]; then + echo " + This script will generate self-signed ssl certificates, they will be written to the current directory + Options available: + -h to set Hostnames (can be used multiple times) + -i to set IP Adresses (can be used multiple times) + -m (optional) activates a magic mode where the script try to find Hostnames and IPs of the current Host + -d (optional) Number of Days the certificate is valid (default=365) + -s (optional) The secret that is used for the crypted key (default=secret) + " + exit 0 + +fi + +if [ -n "${MMODE}" ]; then + echo "Magic Mode is on + this will try to find the hostname and IP of host where this script is executed. + it will then add this to the list of possible Hostnames and IPs + + If you get an error with the Magic Mode then retry with only one hostname set via -h option + + " + + HOSTNAME_BIN=$(type -p hostname) + + # possible addition + # + # try if dig is installed and check the hostname and ip resolve + # dig_bin=$(which dig) + + if [ -n "${HOSTNAME_BIN}" ];then + HNAME+=("$(hostname -s)") + HNAME+=("$(hostname -A)") + # add localhost as hostname to easy up debugging + HNAME+=(localhost) + # try if hostname -I returns the IP, if not + # nasty workaround two steps because the array will get + # entries that can't be parsed out correct + GETIP=$({hostname -I 2>/dev/null || echo "127.0.0.1") + HIP+=($(echo $GETIP | tr -d '[:blank:]')) + else + echo "The command hostname can't be found + aborting Magic mode + please use manual mode and provide at least one hostname with -h + " + exit 1 + fi + + # take all IP Adresses returned by the command IP into the list + # first check if all binaries are present that are needed + # (when only bash build-ins are needed would be awesome) + IPCMD=$(type -p ip) + GRPCMD=$(type -p grep) + AWKCMD=$(type -p awk) + CUTCMD=$(type -p cut) + + if [ -n "${IPCMD}" ] && [ -n "${GRPCMD}" ] && [ -n "${AWKCMD}" ] && [ -n "${CUTCMD}" ]; then + # to avoid error output in the array 2>/dev/null + # every IP that is returned will be added to the array + # ip addr show | grep 'inet ' | awk '{ print $2}' | cut -d"/" -f1 + HIP+=($("${IPCMD}" addr show 2>/dev/null | "${GRPCMD}" 'inet ' 2>/dev/null| "${AWKCMD}" '{print $2}' 2>/dev/null| "${CUTCMD}" -d"/" -f1 2>/dev/null)) + fi +fi + +if [ -z "${HNAME}" ]; then + echo "please provide hostname (-h) at least once. Try -? for help."; + exit 1; +fi + +if [ -z "${OPSSLBIN}" ]; then + echo "no openssl detected aborting" + exit 1; +fi + +# set localhost IP if no other set +if [ -z "${HIP}" ]; then + HIP+=(127.0.0.1) +fi + +# if no VALIDDAYS are set, default 365 +if [ -z "${VALIDDAYS}" ]; then + VALIDDAYS=365 +fi + +# if no Key provided, set default secret +if [ -z "${KEYSECRET}" ]; then + KEYSECRET=secret +fi + + +# sort array entries and make them uniq +NAMES=($(printf "DNS:%q\n" ${HNAME[@]} | sort -u)) +IPADD=($(printf "IP:%q\n" ${HIP[@]} | sort -u)) + +# print each elemet of both arrays with comma seperator +# and create a string from the array content +SUBALT=$(IFS=','; echo "${NAMES[*]},${IPADD[*]}") + +#### output some informatione +echo "This script will generate a SSL certificate with the following settings: +CN Hostname = ${HNAME} +subjectAltName = ${SUBALT} +" +# --------------------------- + +local_openssl_config=" +[ req ] +prompt = no +distinguished_name = req_distinguished_name +x509_extensions = san_self_signed +[ req_distinguished_name ] +CN=${HNAME} +[ san_self_signed ] +subjectAltName = ${SUBALT} +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +basicConstraints = CA:true +" + + ${OPSSLBIN} req \ + -newkey rsa:2048 -nodes \ + -keyout "${HNAME}.pkcs5-plain.key.pem" \ + -x509 -sha256 -days ${VALIDDAYS} \ + -config <(echo "$local_openssl_config") \ + -out "${HNAME}.cert.pem" 2>openssl_error.log || { echo -e "ERROR !\nOpenSSL returns an error, sorry this script will not work \n Possible reason: the openssl version is to old and does not support self signed san certificates \n Check openssl_error.log in your current directory for details"; exit 1; } + +${OPSSLBIN} pkcs8 -in "${HNAME}.pkcs5-plain.key.pem" -topk8 -nocrypt -out "${HNAME}.pkcs8-plain.key.pem" +${OPSSLBIN} pkcs8 -in "${HNAME}.pkcs5-plain.key.pem" -topk8 -passout pass:"${KEYSECRET}" -out "${HNAME}.pkcs8-encrypted.key.pem" + +echo "the following files are written to the current directory:" +echo " - ${HNAME}.pkcs5-plain.key.pem" +echo " - ${HNAME}.pkcs8-plain.key.pem" +echo " - ${HNAME}.pkcs8-encrypted.key.pem" +echo " with the password: ${KEYSECRET}" +echo "" + +rm openssl_error.log + +#EOF diff -Nru graypy-1.1.3/tests/config/docker-compose.yml graypy-2.1.0/tests/config/docker-compose.yml --- graypy-1.1.3/tests/config/docker-compose.yml 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/config/docker-compose.yml 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,31 @@ +version: '2' +services: + mongo: + image: "mongo:3" + elasticsearch: + image: "elasticsearch:2" + command: "elasticsearch -Des.cluster.name='graylog'" + graylog: + image: graylog2/server + environment: + GRAYLOG_PASSWORD_SECRET: CVanHILkuYhsxE50BrNR6FFt75rS3h0V2uUlHxAshGB90guZznEoDxN7zhPx6Bcn61mfhY2T5r0PRkZVwowsTkHU2rBZnv0d + GRAYLOG_ROOT_PASSWORD_SHA2: 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 + GRAYLOG_WEB_ENDPOINT_URI: http://127.0.0.1:9000/api + GRAYLOG_CONTENT_PACKS_AUTO_LOAD: grok-patterns.json,inputs.json + GRAYLOG_ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + volumes: + - ./inputs.json:/usr/share/graylog/data/contentpacks/inputs.json + - ./localhost.cert.pem:/usr/share/graylog/data/cert.pem + - ./localhost.pkcs8-encrypted.key.pem:/usr/share/graylog/data/key.pem + links: + - mongo + - elasticsearch + depends_on: + - mongo + - elasticsearch + ports: + - "9000:9000" + - "12201:12201/tcp" + - "12202:12202/udp" + - "12203:12203" + - "12204:12204/tcp" diff -Nru graypy-1.1.3/tests/config/inputs.json graypy-2.1.0/tests/config/inputs.json --- graypy-1.1.3/tests/config/inputs.json 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/config/inputs.json 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,48 @@ +{ + "inputs": [ + { + "title": "tcp", + "configuration": { + "bind_address": "0.0.0.0", + "port": 12201 + }, + "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput", + "global": true + }, + { + "title": "udp", + "configuration": { + "bind_address": "0.0.0.0", + "port": 12202 + }, + "type": "org.graylog2.inputs.gelf.udp.GELFUDPInput", + "global": true + }, + { + "title": "http", + "configuration": { + "bind_address": "0.0.0.0", + "port": 12203 + }, + "type": "org.graylog2.inputs.gelf.http.GELFHttpInput", + "global": true + }, + { + "title": "tls", + "configuration": { + "bind_address": "0.0.0.0", + "port": 12204, + "tls_enable": true, + "tls_cert_file": "/usr/share/graylog/data/cert.pem", + "tls_key_file": "/usr/share/graylog/data/key.pem", + "tls_key_password": "secret" + }, + "type": "org.graylog2.inputs.gelf.tcp.GELFTCPInput", + "global": true + } + ], + "streams": [], + "outputs": [], + "dashboards": [], + "grok_patterns": [] +} \ No newline at end of file diff -Nru graypy-1.1.3/tests/config/start_local_graylog_server.sh graypy-2.1.0/tests/config/start_local_graylog_server.sh --- graypy-1.1.3/tests/config/start_local_graylog_server.sh 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/config/start_local_graylog_server.sh 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# start a local graylog server for integration testing graypy + +# do work within ./test/config directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd ${DIR} + +# create ssl certs for enabling the graylog server to use a +# TLS connection for GELF input +bash create_ssl_certs.sh -h localhost -i 127.0.0.1 + +# start the graylog server docker container +docker-compose -f docker-compose.yml down +docker-compose -f docker-compose.yml up -d + +# wait for the graylog server docker container to start +sleep 40 + +# test that the graylog server docker container is started +curl -u admin:admin 'http://127.0.0.1:9000/api/search/universal/relative?query=test&range=5&fields=message' || true diff -Nru graypy-1.1.3/tests/config/stop_local_graylog_server.sh graypy-2.1.0/tests/config/stop_local_graylog_server.sh --- graypy-1.1.3/tests/config/stop_local_graylog_server.sh 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/config/stop_local_graylog_server.sh 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# stop the local graylog server used for integration testing graypy + +# do work within ./test/config directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd ${DIR} + +docker-compose -f docker-compose.yml down diff -Nru graypy-1.1.3/tests/helper.py graypy-2.1.0/tests/helper.py --- graypy-1.1.3/tests/helper.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/helper.py 2019-09-30 22:34:35.000000000 +0000 @@ -9,8 +9,7 @@ import logging import pytest -from graypy import GELFUDPHandler, GELFTCPHandler, GELFTLSHandler, \ - GELFHTTPHandler +from graypy import GELFUDPHandler, GELFTCPHandler, GELFTLSHandler, GELFHTTPHandler TEST_CERT = "tests/config/localhost.cert.pem" KEY_PASS = "secret" @@ -21,23 +20,45 @@ TEST_TLS_PORT = 12204 -@pytest.fixture(params=[ - GELFTCPHandler("127.0.0.1", TEST_TCP_PORT), - GELFTCPHandler("127.0.0.1", TEST_TCP_PORT, extra_fields=True), - GELFTCPHandler("127.0.0.1", TEST_TCP_PORT, extra_fields=True, debugging_fields=True), - GELFTLSHandler("localhost", TEST_TLS_PORT), - GELFTLSHandler("localhost", TEST_TLS_PORT, validate=True, ca_certs=TEST_CERT), - GELFTLSHandler("127.0.0.1", TEST_TLS_PORT), - GELFTLSHandler("127.0.0.1", TEST_TLS_PORT, validate=True, ca_certs=TEST_CERT), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT, compress=False), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, compress=False), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, extra_fields=True), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, extra_fields=True, compress=False), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, extra_fields=True, debugging_fields=True), - GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, extra_fields=True, debugging_fields=True, compress=False), -]) +@pytest.fixture( + params=[ + GELFTCPHandler("127.0.0.1", TEST_TCP_PORT), + GELFTCPHandler("127.0.0.1", TEST_TCP_PORT, extra_fields=True), + GELFTCPHandler( + "127.0.0.1", TEST_TCP_PORT, extra_fields=True, debugging_fields=True + ), + GELFTLSHandler("localhost", TEST_TLS_PORT), + GELFTLSHandler("localhost", TEST_TLS_PORT, validate=True, ca_certs=TEST_CERT), + GELFTLSHandler("127.0.0.1", TEST_TLS_PORT), + GELFTLSHandler("127.0.0.1", TEST_TLS_PORT, validate=True, ca_certs=TEST_CERT), + GELFHTTPHandler("127.0.0.1", TEST_HTTP_PORT), + GELFHTTPHandler("127.0.0.1", TEST_HTTP_PORT, compress=False), + GELFUDPHandler("127.0.0.1", TEST_UDP_PORT), + GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, compress=False), + # the below handler tests are essentially smoke tests + # that help cover the argument permutations of BaseGELFHandler + GELFUDPHandler( + "127.0.0.1", + TEST_UDP_PORT, + debugging_fields=True, + extra_fields=True, + localname="foobar_localname", + facility="foobar_facility", + level_names=True, + compress=False, + ), + GELFUDPHandler( + "127.0.0.1", + TEST_UDP_PORT, + debugging_fields=True, + extra_fields=True, + fqdn=True, + facility="foobar_facility", + level_names=True, + compress=False, + ), + ] +) def handler(request): return request.param diff -Nru graypy-1.1.3/tests/integration/helper.py graypy-2.1.0/tests/integration/helper.py --- graypy-1.1.3/tests/integration/helper.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/helper.py 2019-09-30 22:34:35.000000000 +0000 @@ -14,19 +14,36 @@ DEFAULT_FIELDS = [ - "message", "full_message", "source", "level", - "func", "file", "line", "module", "logger_name", + "message", + "full_message", + "source", + "level", + "func", + "file", + "line", + "module", + "logger_name", ] -BASE_API_URL = 'http://127.0.0.1:9000/api/search/universal/relative?query=message:"{0}"&range=5&fields=' +BASE_API_URL = 'http://127.0.0.1:9000/api/search/universal/relative?query=message:"{0}"&range=300&fields=' def get_graylog_response(message, fields=None): """Search for a given log message (with possible additional fields) within a local Graylog instance""" fields = fields if fields else [] - api_resp = _get_api_response(message, fields) - return _parse_api_response(api_resp) + tries = 0 + + while True: + try: + return _parse_api_response( + api_response=_get_api_response(message, fields), wanted_message=message + ) + except ValueError: + sleep(2) + if tries == 5: + raise + tries += 1 def _build_api_string(message, fields): @@ -34,19 +51,21 @@ def _get_api_response(message, fields): - sleep(2) url = _build_api_string(message, fields) api_response = requests.get( - url, - auth=("admin", "admin"), - headers={"accept": "application/json"} + url, auth=("admin", "admin"), headers={"accept": "application/json"} ) return api_response -def _parse_api_response(api_response): +def _parse_api_response(api_response, wanted_message): assert api_response.status_code == 200 print(api_response.json()) - messages = api_response.json()["messages"] - assert 1 == len(messages) - return messages[0]["message"] + for message in api_response.json()["messages"]: + if message["message"]["message"] == wanted_message: + return message["message"] + raise ValueError( + "wanted_message: '{}' not within api_response: {}".format( + wanted_message, api_response + ) + ) diff -Nru graypy-1.1.3/tests/integration/test_chunked_logging.py graypy-2.1.0/tests/integration/test_chunked_logging.py --- graypy-1.1.3/tests/integration/test_chunked_logging.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/test_chunked_logging.py 2019-09-30 22:34:35.000000000 +0000 @@ -7,20 +7,29 @@ import pytest -from graypy.handler import SYSLOG_LEVELS, GELFUDPHandler +from graypy.handler import ( + SYSLOG_LEVELS, + GELFUDPHandler, + GELFWarningChunker, + BaseGELFChunker, + GELFTruncatingChunker, +) from tests.helper import TEST_UDP_PORT from tests.integration import LOCAL_GRAYLOG_UP from tests.integration.helper import get_unique_message, get_graylog_response -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") -def test_chunked_logging(): - """Test sending a common usage log that requires chunking to be fully - sent""" +@pytest.mark.parametrize( + "gelf_chunker", [BaseGELFChunker, GELFWarningChunker, GELFTruncatingChunker] +) +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") +def test_chunked_logging(gelf_chunker): + """Test sending a log that requires chunking to be fully sent""" logger = logging.getLogger("test_chunked_logger") - handler = GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, chunk_size=10) + handler = GELFUDPHandler( + "127.0.0.1", TEST_UDP_PORT, gelf_chunker=gelf_chunker(chunk_size=10) + ) logger.addHandler(handler) message = get_unique_message() logger.error(message) diff -Nru graypy-1.1.3/tests/integration/test_common_logging.py graypy-2.1.0/tests/integration/test_common_logging.py --- graypy-1.1.3/tests/integration/test_common_logging.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/test_common_logging.py 2019-09-30 22:34:35.000000000 +0000 @@ -14,8 +14,7 @@ from tests.integration.helper import get_unique_message, get_graylog_response -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_common_logging(logger): """Test sending a common usage log""" message = get_unique_message() diff -Nru graypy-1.1.3/tests/integration/test_debugging_fields.py graypy-2.1.0/tests/integration/test_debugging_fields.py --- graypy-1.1.3/tests/integration/test_debugging_fields.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/test_debugging_fields.py 2019-09-30 22:34:35.000000000 +0000 @@ -6,40 +6,58 @@ import pytest -from tests.helper import logger, TEST_CERT, TEST_TCP_PORT, TEST_HTTP_PORT, \ - TEST_TLS_PORT, TEST_UDP_PORT +from tests.helper import ( + logger, + TEST_CERT, + TEST_TCP_PORT, + TEST_HTTP_PORT, + TEST_TLS_PORT, + TEST_UDP_PORT, +) from tests.integration import LOCAL_GRAYLOG_UP from tests.integration.helper import get_graylog_response, get_unique_message -from graypy import GELFUDPHandler, GELFTCPHandler, GELFTLSHandler, \ - GELFHTTPHandler +from graypy import GELFUDPHandler, GELFTCPHandler, GELFTLSHandler, GELFHTTPHandler -@pytest.fixture(params=[ - GELFTCPHandler('127.0.0.1', TEST_TCP_PORT, debugging_fields=True), - GELFUDPHandler('127.0.0.1', TEST_UDP_PORT, debugging_fields=True), - GELFUDPHandler('127.0.0.1', TEST_UDP_PORT, compress=False, debugging_fields=True), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT, debugging_fields=True), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT, compress=False, debugging_fields=True), - GELFTLSHandler('127.0.0.1', TEST_TLS_PORT, debugging_fields=True), - GELFTLSHandler('127.0.0.1', TEST_TLS_PORT, debugging_fields=True, validate=True, ca_certs=TEST_CERT), -]) +@pytest.fixture( + params=[ + GELFTCPHandler("127.0.0.1", TEST_TCP_PORT, debugging_fields=True), + GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, debugging_fields=True), + GELFUDPHandler( + "127.0.0.1", TEST_UDP_PORT, compress=False, debugging_fields=True + ), + GELFHTTPHandler("127.0.0.1", TEST_HTTP_PORT, debugging_fields=True), + GELFHTTPHandler( + "127.0.0.1", TEST_HTTP_PORT, compress=False, debugging_fields=True + ), + GELFTLSHandler("127.0.0.1", TEST_TLS_PORT, debugging_fields=True), + GELFTLSHandler( + "127.0.0.1", + TEST_TLS_PORT, + debugging_fields=True, + validate=True, + ca_certs=TEST_CERT, + ), + ] +) def handler(request): return request.param -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_debug_mode(logger): message = get_unique_message() logger.error(message) - graylog_response = get_graylog_response(message, fields=["function", "pid", "thread_name"]) - assert message == graylog_response['message'] + graylog_response = get_graylog_response( + message, fields=["function", "pid", "thread_name"] + ) + assert message == graylog_response["message"] assert "long_message" not in graylog_response assert "timestamp" in graylog_response - assert graylog_response['file'].endswith("test_debugging_fields.py") - assert 'test_debug_mode' == graylog_response['function'] - assert 'line' in graylog_response + assert graylog_response["file"].endswith("test_debugging_fields.py") + assert "test_debug_mode" == graylog_response["function"] + assert "line" in graylog_response assert "file" in graylog_response assert "pid" in graylog_response assert "thread_name" in graylog_response diff -Nru graypy-1.1.3/tests/integration/test_extra_fields.py graypy-2.1.0/tests/integration/test_extra_fields.py --- graypy-1.1.3/tests/integration/test_extra_fields.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/test_extra_fields.py 2019-09-30 22:34:35.000000000 +0000 @@ -5,39 +5,51 @@ import logging import pytest -from graypy import GELFTLSHandler, GELFTCPHandler, GELFUDPHandler, \ - GELFHTTPHandler +from graypy import GELFTLSHandler, GELFTCPHandler, GELFUDPHandler, GELFHTTPHandler -from tests.helper import TEST_CERT, TEST_TCP_PORT, TEST_HTTP_PORT, \ - TEST_TLS_PORT, TEST_UDP_PORT +from tests.helper import ( + TEST_CERT, + TEST_TCP_PORT, + TEST_HTTP_PORT, + TEST_TLS_PORT, + TEST_UDP_PORT, +) from tests.integration import LOCAL_GRAYLOG_UP from tests.integration.helper import get_unique_message, get_graylog_response class DummyFilter(logging.Filter): def filter(self, record): - record.ozzy = 'diary of a madman' + record.ozzy = "diary of a madman" record.van_halen = 1984 record.id = 42 return True -@pytest.fixture(params=[ - GELFTCPHandler('127.0.0.1', TEST_TCP_PORT, extra_fields=True), - GELFUDPHandler('127.0.0.1', TEST_UDP_PORT, extra_fields=True), - GELFUDPHandler('127.0.0.1', TEST_UDP_PORT, compress=False, extra_fields=True), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT, extra_fields=True), - GELFHTTPHandler('127.0.0.1', TEST_HTTP_PORT, compress=False, extra_fields=True), - GELFTLSHandler('127.0.0.1', TEST_TLS_PORT, extra_fields=True), - GELFTLSHandler('127.0.0.1', TEST_TLS_PORT, validate=True, ca_certs=TEST_CERT, extra_fields=True), -]) +@pytest.fixture( + params=[ + GELFTCPHandler("127.0.0.1", TEST_TCP_PORT, extra_fields=True), + GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, extra_fields=True), + GELFUDPHandler("127.0.0.1", TEST_UDP_PORT, compress=False, extra_fields=True), + GELFHTTPHandler("127.0.0.1", TEST_HTTP_PORT, extra_fields=True), + GELFHTTPHandler("127.0.0.1", TEST_HTTP_PORT, compress=False, extra_fields=True), + GELFTLSHandler("127.0.0.1", TEST_TLS_PORT, extra_fields=True), + GELFTLSHandler( + "127.0.0.1", + TEST_TLS_PORT, + validate=True, + ca_certs=TEST_CERT, + extra_fields=True, + ), + ] +) def handler(request): return request.param @pytest.yield_fixture def logger(handler): - logger = logging.getLogger('test') + logger = logging.getLogger("test") dummy_filter = DummyFilter() logger.addFilter(dummy_filter) logger.addHandler(handler) @@ -46,16 +58,15 @@ logger.removeFilter(dummy_filter) -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_dynamic_fields(logger): message = get_unique_message() logger.error(message) - graylog_response = get_graylog_response(message, fields=['ozzy', 'van_halen']) - assert message == graylog_response['message'] + graylog_response = get_graylog_response(message, fields=["ozzy", "van_halen"]) + assert message == graylog_response["message"] assert "long_message" not in graylog_response assert "timestamp" in graylog_response - assert 'diary of a madman' == graylog_response['ozzy'] - assert 1984 == graylog_response['van_halen'] - assert 42 != graylog_response['_id'] - assert 'id' not in graylog_response + assert "diary of a madman" == graylog_response["ozzy"] + assert 1984 == graylog_response["van_halen"] + assert 42 != graylog_response["_id"] + assert "id" not in graylog_response diff -Nru graypy-1.1.3/tests/integration/test_status_issue.py graypy-2.1.0/tests/integration/test_status_issue.py --- graypy-1.1.3/tests/integration/test_status_issue.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/integration/test_status_issue.py 2019-09-30 22:34:35.000000000 +0000 @@ -18,8 +18,7 @@ from tests.integration.helper import get_unique_message, get_graylog_response -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_non_status_field_log(logger): message = get_unique_message() logger.error(message, extra={"foo": "bar"}) @@ -30,8 +29,7 @@ assert "bar" == graylog_response["foo"] -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_status_field_issue(logger): message = get_unique_message() logger.error(message, extra={"status": "OK"}) @@ -42,8 +40,7 @@ assert "OK" == graylog_response["status"] -@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, - reason="local Graylog instance not up") +@pytest.mark.skipif(not LOCAL_GRAYLOG_UP, reason="local Graylog instance not up") def test_status_field_issue_multi(logger): message = get_unique_message() logger.error(message, extra={"foo": "bar", "status": "OK"}) diff -Nru graypy-1.1.3/tests/unit/__init__.py graypy-2.1.0/tests/unit/__init__.py --- graypy-1.1.3/tests/unit/__init__.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/unit/__init__.py 2019-09-30 22:34:35.000000000 +0000 @@ -5,6 +5,6 @@ .. note:: - These tests mock sending to Graylog and thus, do not require a localhost - instance of Graylog to successfully run. + These tests mock sending to Graylog, thus, do not require a local instance + of Graylog to successfully run. """ diff -Nru graypy-1.1.3/tests/unit/test_chunking.py graypy-2.1.0/tests/unit/test_chunking.py --- graypy-1.1.3/tests/unit/test_chunking.py 1970-01-01 00:00:00.000000000 +0000 +++ graypy-2.1.0/tests/unit/test_chunking.py 2019-09-30 22:34:35.000000000 +0000 @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""pytests for various GELF UDP message chunkers""" + +import json +import logging +import struct +import zlib + +import pytest + +from graypy.handler import ( + GELFTruncatingChunker, + GELFWarningChunker, + BaseGELFChunker, + BaseGELFHandler, + SYSLOG_LEVELS, + GELFChunkOverflowWarning, + GELFTruncationFailureWarning, +) + + +@pytest.mark.parametrize( + "gelf_chunker", [BaseGELFChunker, GELFWarningChunker, GELFTruncatingChunker] +) +def test_gelf_chunking(gelf_chunker): + """Test various GELF chunkers""" + message = b"12345" + header = b"\x1e\x0f" + chunks = list(gelf_chunker(chunk_size=2).chunk_message(message)) + expected = [ + (struct.pack("b", 0), struct.pack("b", 3), b"12"), + (struct.pack("b", 1), struct.pack("b", 3), b"34"), + (struct.pack("b", 2), struct.pack("b", 3), b"5"), + ] + + assert len(chunks) == len(expected) + + for index, chunk in enumerate(chunks): + expected_index, expected_chunks_count, expected_chunk = expected[index] + assert header == chunk[:2] + assert expected_index == chunk[10:11] + assert expected_chunks_count == chunk[11:12] + assert expected_chunk == chunk[12:] + + +def rebuild_gelf_bytes_from_udp_chunks(chunks): + gelf_bytes = b"" + bsize = len(chunks[0]) + for chunk in chunks: + if len(chunk) < bsize: + gelf_bytes += chunk[-(bsize - len(chunk)) :] + else: + gelf_bytes += chunk[((2 + struct.calcsize("QBB")) - len(chunk)) :] + return gelf_bytes + + +@pytest.mark.parametrize( + "gelf_chunker", [BaseGELFChunker, GELFWarningChunker, GELFTruncatingChunker] +) +def test_gelf_chunkers(gelf_chunker): + message = BaseGELFHandler().makePickle( + logging.LogRecord( + "test_gelf_chunkers", logging.INFO, None, None, "1" * 10, None, None + ) + ) + chunks = list(gelf_chunker(chunk_size=2).chunk_message(message)) + assert len(chunks) <= 128 + + +@pytest.mark.parametrize( + "gelf_chunker", [BaseGELFChunker, GELFWarningChunker, GELFTruncatingChunker] +) +def test_gelf_chunkers_overflow(gelf_chunker): + message = BaseGELFHandler().makePickle( + logging.LogRecord( + "test_gelf_chunkers_overflow", + logging.INFO, + None, + None, + "1" * 1000, + None, + None, + ) + ) + chunks = list(gelf_chunker(chunk_size=1).chunk_message(message)) + assert len(chunks) <= 128 + + +def test_chunk_overflow_truncate_uncompressed(): + message = BaseGELFHandler(compress=False).makePickle( + logging.LogRecord( + "test_chunk_overflow_truncate_uncompressed", + logging.INFO, + None, + None, + "1" * 1000, + None, + None, + ) + ) + with pytest.warns(GELFChunkOverflowWarning): + chunks = list( + GELFTruncatingChunker(chunk_size=2, compress=False).chunk_message(message) + ) + assert len(chunks) <= 128 + payload = rebuild_gelf_bytes_from_udp_chunks(chunks).decode("UTF-8") + glef_json = json.loads(payload) + assert glef_json["_chunk_overflow"] is True + assert glef_json["short_message"] in "1" * 1000 + assert glef_json["level"] == SYSLOG_LEVELS.get(logging.ERROR, logging.ERROR) + + +def test_chunk_overflow_truncate_compressed(): + message = BaseGELFHandler(compress=True).makePickle( + logging.LogRecord( + "test_chunk_overflow_truncate_compressed", + logging.INFO, + None, + None, + "123412345" * 5000, + None, + None, + ) + ) + with pytest.warns(GELFChunkOverflowWarning): + chunks = list( + GELFTruncatingChunker(chunk_size=2, compress=True).chunk_message(message) + ) + assert len(chunks) <= 128 + payload = zlib.decompress(rebuild_gelf_bytes_from_udp_chunks(chunks)).decode( + "UTF-8" + ) + glef_json = json.loads(payload) + assert glef_json["_chunk_overflow"] is True + assert glef_json["short_message"] in "123412345" * 5000 + assert glef_json["level"] == SYSLOG_LEVELS.get(logging.ERROR, logging.ERROR) + + +def test_chunk_overflow_truncate_fail(): + message = BaseGELFHandler().makePickle( + logging.LogRecord( + "test_chunk_overflow_truncate_fail", + logging.INFO, + None, + None, + "1" * 1000, + None, + None, + ) + ) + with pytest.warns(GELFTruncationFailureWarning): + list(GELFTruncatingChunker(1).chunk_message(message)) + + +def test_chunk_overflow_truncate_fail_large_inherited_field(): + message = BaseGELFHandler( + facility="this is a really long facility" * 5000 + ).makePickle( + logging.LogRecord( + "test_chunk_overflow_truncate_fail", + logging.INFO, + None, + None, + "reasonable message", + None, + None, + ) + ) + with pytest.warns(GELFTruncationFailureWarning): + list(GELFTruncatingChunker(2).chunk_message(message)) diff -Nru graypy-1.1.3/tests/unit/test_ExcludeFilter.py graypy-2.1.0/tests/unit/test_ExcludeFilter.py --- graypy-1.1.3/tests/unit/test_ExcludeFilter.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/unit/test_ExcludeFilter.py 2019-09-30 22:34:35.000000000 +0000 @@ -22,19 +22,19 @@ def test_valid_name(name): """Test constructing :class:`graypy.rabbitmq.ExcludeFilter` with a valid ``name`` argument""" - filter = ExcludeFilter(name) - assert filter - assert name == filter.name - assert len(name) == filter.nlen + exclude_filter = ExcludeFilter(name) + assert exclude_filter + assert name == exclude_filter.name + assert len(name) == exclude_filter.nlen def test_non_filtering_record(): - filter = ExcludeFilter("NOT" + MOCK_LOG_RECORD_NAME) - assert filter.filter(MOCK_LOG_RECORD) - assert MOCK_LOG_RECORD.name != filter.name + exclude_filter = ExcludeFilter("NOT" + MOCK_LOG_RECORD_NAME) + assert exclude_filter.filter(MOCK_LOG_RECORD) + assert MOCK_LOG_RECORD.name != exclude_filter.name def test_filtering_record(): - filter = ExcludeFilter(MOCK_LOG_RECORD_NAME) - assert not filter.filter(MOCK_LOG_RECORD) - assert MOCK_LOG_RECORD.name == filter.name + exclude_filter = ExcludeFilter(MOCK_LOG_RECORD_NAME) + assert not exclude_filter.filter(MOCK_LOG_RECORD) + assert MOCK_LOG_RECORD.name == exclude_filter.name diff -Nru graypy-1.1.3/tests/unit/test_handler.py graypy-2.1.0/tests/unit/test_handler.py --- graypy-1.1.3/tests/unit/test_handler.py 2019-07-15 23:22:38.000000000 +0000 +++ graypy-2.1.0/tests/unit/test_handler.py 2019-09-30 22:34:35.000000000 +0000 @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""pytests for the formatting and construction of Graylog GLEF logs by graypy +"""pytests for the formatting and construction of GELF logs by the graypy +logging handlers .. note:: @@ -13,15 +14,13 @@ import json import logging import socket -import struct import sys import zlib import mock import pytest -from graypy.handler import BaseGELFHandler, GELFHTTPHandler, GELFTLSHandler, \ - ChunkedGELF +from graypy.handler import BaseGELFHandler, GELFHTTPHandler, GELFTLSHandler from tests.helper import handler, logger, formatted_logger from tests.unit.helper import MOCK_LOG_RECORD, MOCK_LOG_RECORD_NAME @@ -50,7 +49,9 @@ # TODO: this is inaccurate solution for mocking non-send handlers if isinstance(arg, logging.LogRecord): - return json.loads(BaseGELFHandler(compress=False).makePickle(arg).decode("utf-8")) + return json.loads( + BaseGELFHandler(compress=False).makePickle(arg).decode("utf-8") + ) try: return json.loads(zlib.decompress(arg).decode("utf-8")) except zlib.error: # we have a uncompress message @@ -60,14 +61,19 @@ return json.loads(arg[:-1].decode("utf-8")) -@pytest.mark.parametrize("message,expected", [ - (u"\u20AC", u"\u20AC"), - (u"\u20AC".encode("utf-8"), u"\u20AC"), - (b"\xc3", UNICODE_REPLACEMENT), - (["a", b"\xc3"], ["a", UNICODE_REPLACEMENT]), -]) +@pytest.mark.parametrize( + "message,expected", + [ + (u"\u20AC", u"\u20AC"), + (u"\u20AC".encode("utf-8"), u"\u20AC"), + (b"\xc3", UNICODE_REPLACEMENT), + (["a", b"\xc3"], ["a", UNICODE_REPLACEMENT]), + ], +) def test_pack(message, expected): - assert expected == json.loads(BaseGELFHandler._pack_gelf_dict(message).decode("utf-8")) + assert expected == json.loads( + BaseGELFHandler._pack_gelf_dict(message).decode("utf-8") + ) def test_manual_exc_info_handler(logger, mock_send): @@ -83,8 +89,9 @@ # GELFHTTPHandler mocking does not complete the stacktrace # thus a missing \n - assert arg["full_message"].endswith("SyntaxError: Syntax error") or \ - arg["full_message"].endswith("SyntaxError: Syntax error\n") + assert arg["full_message"].endswith("SyntaxError: Syntax error") or arg[ + "full_message" + ].endswith("SyntaxError: Syntax error\n") def test_normal_exception_handler(logger, mock_send): @@ -98,8 +105,9 @@ # GELFHTTPHandler mocking does not complete the stacktrace # thus a missing \n - assert arg["full_message"].endswith("SyntaxError: Syntax error") or \ - arg["full_message"].endswith("SyntaxError: Syntax error\n") + assert arg["full_message"].endswith("SyntaxError: Syntax error") or arg[ + "full_message" + ].endswith("SyntaxError: Syntax error\n") def test_unicode(logger, mock_send): @@ -148,7 +156,9 @@ assert "" == decoded["_foo"] -def test_message_to_pickle_serializes_datetime_objects_instead_of_blindly_repring_them(logger, mock_send): +def test_message_to_pickle_serializes_datetime_objects_instead_of_blindly_repring_them( + logger, mock_send +): timestamp = datetime.datetime(2001, 2, 3, 4, 5, 6, 7) logger.error("Log message", extra={"ts": timestamp}) decoded = get_mock_send_arg(mock_send) @@ -202,7 +212,7 @@ """Test constructing :class:`graypy.handler.BaseGELFHandler` with specifying conflicting arguments ``fqdn`` and ``localname``""" with pytest.raises(ValueError): - BaseGELFHandler("127.0.0.1", 12202, fqdn=True, localname="localhost") + BaseGELFHandler(fqdn=True, localname="localhost") def test_invalid_ca_certs(): @@ -218,25 +228,3 @@ with pytest.raises(ValueError): # missing client cert GELFTLSHandler("127.0.0.1", keyfile="/dev/null") - - -def test_glef_chunking(): - """Testing the GELF chunking ability of - :class:`graypy.handler.ChunkedGELF`""" - message = b'12345' - header = b'\x1e\x0f' - chunks = list(ChunkedGELF(message, 2).__iter__()) - expected = [ - (struct.pack('b', 0), struct.pack('b', 3), b'12'), - (struct.pack('b', 1), struct.pack('b', 3), b'34'), - (struct.pack('b', 2), struct.pack('b', 3), b'5') - ] - - assert len(chunks) == len(expected) - - for index, chunk in enumerate(chunks): - expected_index, expected_chunks_count, expected_chunk = expected[index] - assert header == chunk[:2] - assert expected_index == chunk[10:11] - assert expected_chunks_count == chunk[11:12] - assert expected_chunk == chunk[12:]