diff -Nru urlwatch-2.23/CHANGELOG.md urlwatch-2.24/CHANGELOG.md --- urlwatch-2.23/CHANGELOG.md 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/CHANGELOG.md 2021-11-07 08:34:54.000000000 +0000 @@ -4,6 +4,28 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [2.24] -- 2021-11-07 + +### Added + +- The Telegram reporter has gained two new options: + - `silent`: Receive message notification without sound + - `monospace`: Format message in monospace style +- Support for running a subset of jobs by specifying their index on the command line + +### Changed + +- Migrated CI pipeline from Travis CI to Github Actions +- `user_visible_url` can now be specified for all job types (#654, by kongomongo) +- Added a `remove-duplicate-lines` filter. +- Added a `csv2text` filter. +- Set envelope from (`-f` option) when sending emails using `sendmail` +- It is now possible to override the HTTP `method` when `data` is set on a URL job + +### Fixed + +- Fix UnboundLocalError when SMTP auth is enabled, but keyring is not installed (#667) + ## [2.23] -- 2021-04-10 ### Added diff -Nru urlwatch-2.23/debian/changelog urlwatch-2.24/debian/changelog --- urlwatch-2.23/debian/changelog 2021-10-03 16:14:15.000000000 +0000 +++ urlwatch-2.24/debian/changelog 2021-11-12 15:15:01.000000000 +0000 @@ -1,3 +1,13 @@ +urlwatch (2.24-1) unstable; urgency=medium + + * New upstream release + * Update patches to new version + * Add autopkgtest for python + * Update pybuild setup to run all upstream tests + * Removed Built-using following lintian alert built-using-field-on-arch-all-package + + -- Maxime Werlen Fri, 12 Nov 2021 16:15:01 +0100 + urlwatch (2.23-2) unstable; urgency=medium * Updated Standards-Version to 4.6.0 diff -Nru urlwatch-2.23/debian/control urlwatch-2.24/debian/control --- urlwatch-2.23/debian/control 2021-10-03 16:14:10.000000000 +0000 +++ urlwatch-2.24/debian/control 2021-11-12 15:11:21.000000000 +0000 @@ -24,6 +24,7 @@ Homepage: https://thp.io/2008/urlwatch/ Vcs-Git: https://salsa.debian.org/mwerlen/urlwatch.git Vcs-Browser: https://salsa.debian.org/mwerlen/urlwatch +Testsuite: autopkgtest-pkg-python Rules-Requires-Root: no X-Python3-Version: >= 3.6 @@ -33,8 +34,6 @@ ${misc:Depends}, ${python3:Depends}, ${sphinxdoc:Depends}, -Built-Using: - ${sphinxdoc:Built-Using} Recommends: lynx, html2text, diff -Nru urlwatch-2.23/debian/patches/RemoveDependenciesFromReadme.diff urlwatch-2.24/debian/patches/RemoveDependenciesFromReadme.diff --- urlwatch-2.23/debian/patches/RemoveDependenciesFromReadme.diff 2020-12-19 13:28:29.000000000 +0000 +++ urlwatch-2.24/debian/patches/RemoveDependenciesFromReadme.diff 2021-11-12 13:50:59.000000000 +0000 @@ -1,13 +1,13 @@ -Description: Removes unnecessary parts from Readme.md (travis CI status) +Description: Removes unnecessary parts from Readme.md (github actions status) Author: Maxime Werlen Forwarded: not-needed -Last-Update: 2020-12-19 +Last-Update: 2021-11-12 --- This patch header follows DEP-3: https://dep.debian.net/deps/dep3/ --- a/README.md +++ b/README.md @@ -1,8 +1,3 @@ --[![Build Status](https://travis-ci.org/thp/urlwatch.svg)](https://travis-ci.org/thp/urlwatch) +-[![Unit Tests](https://github.com/thp/urlwatch/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/thp/urlwatch/actions/workflows/unit-tests.yml) -[![Packaging status](https://repology.org/badge/tiny-repos/urlwatch.svg)](https://repology.org/metapackage/urlwatch/versions) -[![PyPI version](https://badge.fury.io/py/urlwatch.svg)](https://badge.fury.io/py/urlwatch) - diff -Nru urlwatch-2.23/debian/patches/removeNonSupportedTestData urlwatch-2.24/debian/patches/removeNonSupportedTestData --- urlwatch-2.23/debian/patches/removeNonSupportedTestData 2021-04-14 16:03:23.000000000 +0000 +++ urlwatch-2.24/debian/patches/removeNonSupportedTestData 2021-11-12 14:39:40.000000000 +0000 @@ -4,12 +4,12 @@ to avoid detection. This does not affect the resulting HTML output. Author: Maxime Werlen Forwarded: not-needed -Last-Update: 2020-07-30 +Last-Update: 2021-11-12 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/docs/source/filters.rst +++ b/docs/source/filters.rst -@@ -283,7 +283,7 @@ +@@ -285,7 +285,7 @@ This filter *must* be the first filter in a chain of filters, since it consumes binary data and outputs text data. @@ -18,7 +18,7 @@ url: https://example.net/pdf-test.pdf filter: -@@ -293,7 +293,7 @@ +@@ -295,7 +295,7 @@ If the PDF file is password protected, you can specify its password: @@ -27,7 +27,16 @@ url: https://example.net/pdf-test-password.pdf filter: -@@ -528,7 +528,7 @@ +@@ -419,7 +419,7 @@ + + Alternatively, ``jq`` can be used for filtering: + +-.. code:: yaml ++.. code:: Yaml + + url: https://api.github.com/repos/voxpupuli/puppet-rundeck/tags + filter: +@@ -556,7 +556,7 @@ This filter *must* be the first filter in a chain of filters, since it consumes binary data and outputs text data. @@ -36,7 +45,7 @@ url: https://example.net/ocr-test.png filter: -@@ -552,7 +552,7 @@ +@@ -580,7 +580,7 @@ .. _jq: https://stedolan.github.io/jq/ .. _jq Python module: https://github.com/mwilliamson/jq.py diff -Nru urlwatch-2.23/debian/rules urlwatch-2.24/debian/rules --- urlwatch-2.23/debian/rules 2020-12-19 17:00:50.000000000 +0000 +++ urlwatch-2.24/debian/rules 2021-11-12 14:49:14.000000000 +0000 @@ -5,7 +5,8 @@ # Documentation snippets are unit-tested, thus tests need access to documentation sources export PYBUILD_BEFORE_TEST=cp -r {dir}/docs {build_dir}/../; \ - cp -r {dir}/lib/urlwatch/tests {build_dir}/urlwatch/ + cp -r {dir}/lib/urlwatch/tests {build_dir}/urlwatch/; \ + cp -r {dir}/share {build_dir}/../ export PYBUILD_AFTER_TEST=rm -rf {build_dir}/../docs {build_dir}/urlwatch/tests override_dh_sphinxdoc: diff -Nru urlwatch-2.23/docs/source/advanced.rst urlwatch-2.24/docs/source/advanced.rst --- urlwatch-2.23/docs/source/advanced.rst 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/docs/source/advanced.rst 2021-11-07 08:34:54.000000000 +0000 @@ -312,3 +312,49 @@ filter: - grep: "Thing B" + +Running a subset of jobs +------------------------ + +To run one or more specific jobs instead of all known jobs, provide +the job index numbers to the urlwatch command. For example, to run +jobs with index 2, 4, and 7: + +.. code-block:: bash + + urlwatch 2 4 7 + + +Sending HTML form data using POST +--------------------------------- + +To simulate submitting a HTML form using the POST method, you can pass +the form fields in the ``data`` field of the job description: + +.. code-block:: yaml + + name: "My POST Job" + url: http://example.com/foo + data: + username: "foo" + password: "bar" + submit: "Send query" + +By default, the request will use the HTTP ``POST`` method, and the +``Content-type`` will be set to ``application/x-www-form-urlencoded``. + + +Sending arbitrary data using HTTP PUT +------------------------------------- + +It is possible to customize the HTTP method and ``Content-type`` header, +allowing you to send arbitrary requests to the server: + +.. code-block:: yaml + + name: "My PUT Request" + url: http://example.com/item/new + method: PUT + headers: + Content-type: application/json + data: '{"foo": true}' diff -Nru urlwatch-2.23/docs/source/conf.py urlwatch-2.24/docs/source/conf.py --- urlwatch-2.23/docs/source/conf.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/docs/source/conf.py 2021-11-07 08:34:54.000000000 +0000 @@ -22,7 +22,7 @@ author = 'Thomas Perl' # The full version, including alpha/beta/rc tags -release = '2.23' +release = '2.24' # -- General configuration --------------------------------------------------- diff -Nru urlwatch-2.23/docs/source/filters.rst urlwatch-2.24/docs/source/filters.rst --- urlwatch-2.23/docs/source/filters.rst 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/docs/source/filters.rst 2021-11-07 08:34:54.000000000 +0000 @@ -69,6 +69,7 @@ - **sha1sum**: Calculate the SHA-1 checksum of the content - **shellpipe**: Filter using a shell command - **sort**: Sort input items +- **remove-duplicate-lines**: Remove duplicate lines (case sensitive) - **strip**: Strip leading and trailing whitespace - **xpath**: Filter XML/HTML using XPath expressions - **jq**: Filter, transform and extract values from JSON @@ -135,9 +136,7 @@ To filter based on an `XPath `__ expression, -you can use the ``xpath`` filter like so (see Microsoft’s `XPath -Examples `__ -page for some other examples): +you can use the ``xpath`` filter like so: .. code:: yaml @@ -149,6 +148,9 @@ element, which in turn must be below the ```` element of the document, stripping out everything else. +See Microsoft’s `XPath Examples `__ page for some other examples. +You can also find an XPath of an ```` node in the Chromium/Google Chrome developer tools by right clicking on the node and selecting ``copy XPath``. + Filtering based on CSS selectors -------------------------------- @@ -301,6 +303,25 @@ password: urlwatchsecret - strip +Dealing with CSV input +---------------------- + +`csv2text` filter can be used to turn CSV data to a prettier textual representation. This is done by +supplying a `format_string` which is a [python format string](https://docs.python.org/3/library/string.html#format-string-syntax). +If the CSV has a header, the format string should use the header names (**lowercased**). Example: + +| Name | Company | +|--|--| +| Smith | Initech | +| Doe | Initech | + +Format string for the above CSV: `Mr {name} works at {company}` (Note the lowercase). +If there is no header row, you will need to use the numeric array notation: +`Mr {0} works at {1}`. +You can also use numeric array on CSV with headers with the flag `ignore_header`. +`has_header` can be used to force use the first line or first ignore the first line as header, +otherwise [csv.Sniffer](https://docs.python.org/3/library/csv.html#csv.Sniffer) +would be used. Sorting of webpage content -------------------------- @@ -396,6 +417,13 @@ - xpath: (//a[contains(@class,"item-title ref-name")])[1] - html2text +Alternatively, ``jq`` can be used for filtering: + +.. code:: yaml + + url: https://api.github.com/repos/voxpupuli/puppet-rundeck/tags + filter: + - jq: '.[0].name' Remove or replace text using regular expressions ------------------------------------------------ @@ -566,4 +594,4 @@ Supports aggregations, selections, and the built-in operators like ``length``. For more information on the operations permitted, see the `jq Manual`_. -.. _jq Manual: https://stedolan.github.io/jq/manual/ \ No newline at end of file +.. _jq Manual: https://stedolan.github.io/jq/manual/ diff -Nru urlwatch-2.23/docs/source/jobs.rst urlwatch-2.24/docs/source/jobs.rst --- urlwatch-2.23/docs/source/jobs.rst 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/docs/source/jobs.rst 2021-11-07 08:34:54.000000000 +0000 @@ -7,7 +7,9 @@ The list of jobs to run are contained in the configuration file ``urls.yaml``, accessed with the command ``urlwatch --edit``, each separated by a line -containing only ``---``. +containing only ``---``. The command ``urlwatch --list`` prints the name +of each job, along with its index number (1,2,3,...) which gets assigned +automatically according to its position in the configuration file. While optional, it is recommended that each job starts with a ``name`` entry: @@ -46,7 +48,6 @@ - ``ignore_http_error_codes``: List of HTTP errors to ignore (see :ref:`advanced_topics`) - ``ignore_timeout_errors``: Do not report errors when the timeout is hit - ``ignore_too_many_redirects``: Ignore redirect loops (see :ref:`advanced_topics`) -- ``user_visible_url``: Different URL to show in reports (e.g. when watched URL is a REST API URL, and you want to show a webpage) (Note: ``url`` implies ``kind: url``) @@ -130,6 +131,7 @@ - ``treat_new_as_changed``: Will treat jobs that don't have any historic data as ``CHANGED`` instead of ``NEW`` (and create a diff for new jobs) - ``compared_versions``: Number of versions to compare for similarity - ``kind`` (redundant): Either ``url``, ``shell`` or ``browser``. Automatically derived from the unique key (``url``, ``command`` or ``navigate``) of the job type +- ``user_visible_url``: Different URL to show in reports (e.g. when watched URL is a REST API URL, and you want to show a webpage) Settings keys for all jobs at once diff -Nru urlwatch-2.23/docs/source/reporters.rst urlwatch-2.24/docs/source/reporters.rst --- urlwatch-2.23/docs/source/reporters.rst 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/docs/source/reporters.rst 2021-11-07 08:34:54.000000000 +0000 @@ -106,6 +106,17 @@ chat_id: '88888888' # the chat id where the messages should be sent enabled: true +Messages can be sent silently (``silent``) if you prefer notifications +with no sounds, and monospace formatted (``monospace``). +By default notifications are not silent and no formatting is done. + +.. code:: yaml + + telegram: + # ... + silent: true # message is sent silently + monospace: true # display message as pre-formatted code block + To set up Telegram, from your Telegram app, chat up BotFather (New Message, Search, “BotFather”), then say ``/newbot`` and follow the instructions. Eventually it will tell you the bot token (in the form @@ -173,7 +184,7 @@ enabled: true embed: true subject: '{count} changes: {jobs}' - + To set up Discord, from your Discord Server settings, select Integration and then create a "New Webhook", give the webhook a name to post under, select a channel, push "Copy Webhook URL" and paste it into the configuration as seen above. Embedded content might be easier to read and identify individual reports. subject preceeds the embedded report and is only used when embed is true. diff -Nru urlwatch-2.23/.github/workflows/unit-tests.yml urlwatch-2.24/.github/workflows/unit-tests.yml --- urlwatch-2.23/.github/workflows/unit-tests.yml 1970-01-01 00:00:00.000000000 +0000 +++ urlwatch-2.24/.github/workflows/unit-tests.yml 2021-11-07 08:34:54.000000000 +0000 @@ -0,0 +1,37 @@ +name: Unit Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + name: Python + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + python -m pip install pycodestyle==2.6.0 pytest + sudo apt install -y build-essential libpoppler-cpp-dev pkg-config python-dev tesseract-ocr + python -m pip install pdftotext docutils pygments pytesseract pillow jq + - name: Test with pytest + run: pytest -v diff -Nru urlwatch-2.23/lib/urlwatch/config.py urlwatch-2.24/lib/urlwatch/config.py --- urlwatch-2.23/lib/urlwatch/config.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/config.py 2021-11-07 08:34:54.000000000 +0000 @@ -75,6 +75,7 @@ parser = argparse.ArgumentParser(description=urlwatch.__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('joblist', metavar='JOB', type=int, nargs="*", help='index of job(s) to run, as numbered according to the --list command. If none specified, then all jobs will be run.') parser.add_argument('--version', action='version', version='%(prog)s {}'.format(urlwatch.__version__)) parser.add_argument('-v', '--verbose', action='store_true', help='show debug output') group = parser.add_argument_group('files and directories') diff -Nru urlwatch-2.23/lib/urlwatch/filters.py urlwatch-2.24/lib/urlwatch/filters.py --- urlwatch-2.23/lib/urlwatch/filters.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/filters.py 2021-11-07 08:34:54.000000000 +0000 @@ -39,6 +39,7 @@ import sys import subprocess import io +import csv from enum import Enum from lxml import etree @@ -330,6 +331,53 @@ method=method, options=options) +class Csv2TextFilter(FilterBase): + """Convert CSV to plaintext""" + + __kind__ = 'csv2text' + + __supported_subfilters__ = { + 'format_message': 'A format string with the headers that will be outputted for each csv ' + 'line (header will be lower-cased)', + 'ignore_header': 'If your format string is number based, but the CSV has headers, ' + 'this flag will force ignoring the header.', + 'has_header': 'If specified and true - use the first line as a header. ' + 'If false - force ignore first line as header (treat it as data). ' + 'If not specified csv.Sniffer will be used.', + } + + __default_subfilter__ = 'format_message' + + def filter(self, data, subfilter): + has_header_config = subfilter.get('has_header') + + if has_header_config is None: + has_header = csv.Sniffer().has_header(data) + else: + has_header = has_header_config + + reader = csv.reader(data.split('\n')) + data_list = list(reader) + header = None + + if has_header: + header = data_list.pop(0) + + header = [i.lower() for i in header] + message = subfilter['format_message'] + ignore_header = subfilter['ignore_header'] + + lines = [] + for i in data_list: + if header and not ignore_header: + legend = dict(zip(header, i)) + lines.append(message.format(**legend)) + else: + lines.append(message.format(*i)) + + return '\n'.join(lines) + + class Pdf2TextFilter(FilterBase): """Convert PDF to plaintext""" # Requires data to be in bytes (not unicode) @@ -809,6 +857,31 @@ return separator.join(sorted(data.split(separator), key=str.casefold, reverse=reverse)) +class RemoveDuplicateLinesFilter(FilterBase): + """Remove duplicate lines""" + + __kind__ = 'remove-duplicate-lines' + + __supported_subfilters__ = { + 'separator': 'Item separator (default: newline)', + } + + __default_subfilter__ = 'separator' + + def filter(self, data, subfilter): + separator = subfilter.get('separator', '\n') + data_lines = data.split(separator) + + def get_unique_lines(lines): + seen = set() + for line in lines: + if line not in seen: + yield line + seen.add(line) + + return separator.join(get_unique_lines(data_lines)) + + class ReverseFilter(FilterBase): """Reverse input items""" diff -Nru urlwatch-2.23/lib/urlwatch/__init__.py urlwatch-2.24/lib/urlwatch/__init__.py --- urlwatch-2.23/lib/urlwatch/__init__.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/__init__.py 2021-11-07 08:34:54.000000000 +0000 @@ -12,5 +12,5 @@ __author__ = 'Thomas Perl ' __license__ = 'BSD' __url__ = 'https://thp.io/2008/urlwatch/' -__version__ = '2.23' +__version__ = '2.24' __user_agent__ = '%s/%s (+https://thp.io/2008/urlwatch/info.html)' % (pkgname, __version__) diff -Nru urlwatch-2.23/lib/urlwatch/jobs.py urlwatch-2.24/lib/urlwatch/jobs.py --- urlwatch-2.23/lib/urlwatch/jobs.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/jobs.py 2021-11-07 08:34:54.000000000 +0000 @@ -180,10 +180,11 @@ class Job(JobBase): __required__ = () - __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed') + __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed', 'user_visible_url') # determine if hyperlink "a" tag is used in HtmlReporter - LOCATION_IS_URL = False + def location_is_url(self): + return re.match("^([a-zA-Z0-9+.-]+)://", self.get_location()) def pretty_name(self): return self.name if self.name else self.get_location() @@ -198,7 +199,7 @@ __optional__ = () def get_location(self): - return self.command + return self.user_visible_url or self.command def retrieve(self, job_state): process = subprocess.Popen(self.command, stdout=subprocess.PIPE, shell=True) @@ -221,9 +222,8 @@ __required__ = ('url',) __optional__ = ('cookies', 'data', 'method', 'ssl_no_verify', 'ignore_cached', 'http_proxy', 'https_proxy', 'headers', 'ignore_connection_errors', 'ignore_http_error_codes', 'encoding', 'timeout', - 'ignore_timeout_errors', 'ignore_too_many_redirects', 'user_visible_url') + 'ignore_timeout_errors', 'ignore_too_many_redirects') - LOCATION_IS_URL = True CHARSET_RE = re.compile('text/(html|plain); charset=([^;]*)') def get_location(self): @@ -251,12 +251,14 @@ headers['Cache-Control'] = 'max-age=172800' headers['Expires'] = email.utils.formatdate() - if self.method is None: - self.method = "GET" if self.data is not None: - self.method = "POST" + if self.method is None: + self.method = "POST" headers['Content-type'] = 'application/x-www-form-urlencoded' - logger.info('Sending POST request to %s', self.url) + logger.info('Sending %s request to %s', self.method, self.url) + + if self.method is None: + self.method = "GET" if self.http_proxy is not None: proxies['http'] = self.http_proxy @@ -366,10 +368,8 @@ __optional__ = ('wait_until',) - LOCATION_IS_URL = True - def get_location(self): - return self.navigate + return self.user_visible_url or self.navigate def main_thread_enter(self): from .browser import BrowserContext diff -Nru urlwatch-2.23/lib/urlwatch/mailer.py urlwatch-2.24/lib/urlwatch/mailer.py --- urlwatch-2.23/lib/urlwatch/mailer.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/mailer.py 2021-11-07 08:34:54.000000000 +0000 @@ -95,6 +95,8 @@ if passwd is None: raise ValueError('No password available in keyring for {}, {}' .format(self.smtp_server, self.smtp_user)) + else: + raise ValueError('SMTP auth is enabled, but insecure_password is not set and keyring is not available') s.login(self.smtp_user, passwd) s.sendmail(msg['From'], msg['To'].split(','), msg.as_string()) @@ -106,7 +108,7 @@ self.sendmail_path = sendmail_path def send(self, msg): - p = subprocess.Popen([self.sendmail_path, '-oi', msg['To']], + p = subprocess.Popen([self.sendmail_path, '-oi', '-f', msg['From'], msg['To']], stdin=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) diff -Nru urlwatch-2.23/lib/urlwatch/reporters.py urlwatch-2.24/lib/urlwatch/reporters.py --- urlwatch-2.23/lib/urlwatch/reporters.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/reporters.py 2021-11-07 08:34:54.000000000 +0000 @@ -183,7 +183,7 @@ for job_state in self.report.get_filtered_job_states(self.job_states): job = job_state.job - if job.LOCATION_IS_URL: + if job.location_is_url(): title = '{pretty_name}' elif job.pretty_name() != job.get_location(): title = '{pretty_name}' @@ -600,11 +600,29 @@ return result + @staticmethod + def _format_body(text: str) -> str: + return "```diff\n{}\n```".format( + text.translate(str.maketrans({'`': r'\`', '\\': r'\\'}))) + def submitToTelegram(self, bot_token, chat_id, text): logger.debug("Sending telegram request to chat id:'{0}'".format(chat_id)) - result = requests.post( - "https://api.telegram.org/bot{0}/sendMessage".format(bot_token), - data={"chat_id": chat_id, "text": text, "disable_web_page_preview": "true"}) + + data = {"chat_id": chat_id, + "text": text, + "disable_notification": self.config.get('silent', False), + "disable_web_page_preview": True} + + if self.config.get('monospace', False): + # all "`" and "\" characters are escaped and text is put inside + # a markdown code block. API docs on formatting messages: + # https://core.telegram.org/bots/api#formatting-options + data.update({ + "text": self._format_body(text), + "parse_mode": "MarkdownV2" + }) + + result = requests.post("https://api.telegram.org/bot{0}/sendMessage".format(bot_token), json=data) try: json_res = result.json() diff -Nru urlwatch-2.23/lib/urlwatch/storage.py urlwatch-2.24/lib/urlwatch/storage.py --- urlwatch-2.23/lib/urlwatch/storage.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/storage.py 2021-11-07 08:34:54.000000000 +0000 @@ -126,6 +126,8 @@ 'enabled': False, 'bot_token': '', 'chat_id': '', + 'monospace': False, + 'silent': False, }, 'slack': { 'enabled': False, diff -Nru urlwatch-2.23/lib/urlwatch/tests/data/filter_documentation_testdata.yaml urlwatch-2.24/lib/urlwatch/tests/data/filter_documentation_testdata.yaml --- urlwatch-2.23/lib/urlwatch/tests/data/filter_documentation_testdata.yaml 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/tests/data/filter_documentation_testdata.yaml 2021-11-07 08:34:54.000000000 +0000 @@ -368,7 +368,33 @@ "company": "Initech" } ] - output: | + output: |- "Senior Data Scientist" "Python Developer" - "TPS Report Analyst" \ No newline at end of file + "TPS Report Analyst" +https://api.github.com/repos/voxpupuli/puppet-rundeck/tags: + input: | + [ + { + "name": "v6.0.0", + "zipball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/zipball/refs/tags/v6.0.0", + "tarball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/tarball/refs/tags/v6.0.0", + "commit": { + "sha": "f539fb125f8fe56cec4f134dd5ef4b5c9be961c5", + "url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/commits/f539fb125f8fe56cec4f134dd5ef4b5c9be961c5" + }, + "node_id": "MDM6UmVmMTgyMDczMjM6cmVmcy90YWdzL3Y2LjAuMA==" + }, + { + "name": "v5.4.0", + "zipball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/zipball/refs/tags/v5.4.0", + "tarball_url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/tarball/refs/tags/v5.4.0", + "commit": { + "sha": "4689cde2189ac38a36c81275fb13925f5c4a8ca2", + "url": "https://api.github.com/repos/voxpupuli/puppet-rundeck/commits/4689cde2189ac38a36c81275fb13925f5c4a8ca2" + }, + "node_id": "MDM6UmVmMTgyMDczMjM6cmVmcy90YWdzL3Y1LjQuMA==" + } + ] + output: |- + "v6.0.0" diff -Nru urlwatch-2.23/lib/urlwatch/tests/test_handler.py urlwatch-2.24/lib/urlwatch/tests/test_handler.py --- urlwatch-2.23/lib/urlwatch/tests/test_handler.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/tests/test_handler.py 2021-11-07 08:34:54.000000000 +0000 @@ -11,7 +11,7 @@ import os from urlwatch import storage -from urlwatch.config import BaseConfig +from urlwatch.config import CommandConfig from urlwatch.storage import YamlConfigStorage, CacheMiniDBStorage from urlwatch.main import Urlwatch from urlwatch.util import import_module_from_source @@ -87,12 +87,10 @@ assert result.total_errors == 0, "Found #{0} code style errors".format(result.total_errors) -class ConfigForTest(BaseConfig): +class ConfigForTest(CommandConfig): def __init__(self, config, urls, cache, hooks, verbose): (prefix, bindir) = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0]))) - super().__init__('urlwatch', os.path.dirname(__file__), config, urls, cache, hooks, verbose) - self.edit = False - self.edit_hooks = False + super().__init__('urlwatch', os.path.dirname(__file__), bindir, prefix, config, urls, hooks, cache, verbose) @contextlib.contextmanager diff -Nru urlwatch-2.23/lib/urlwatch/worker.py urlwatch-2.24/lib/urlwatch/worker.py --- urlwatch-2.23/lib/urlwatch/worker.py 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/lib/urlwatch/worker.py 2021-11-07 08:34:54.000000000 +0000 @@ -51,12 +51,14 @@ def run_jobs(urlwatcher): + if not all(1 <= idx <= len(urlwatcher.jobs) for idx in urlwatcher.urlwatch_config.joblist): + raise ValueError(f'All job indices must be between 1 and {len(urlwatcher.jobs)}: {urlwatcher.urlwatch_config.joblist}') cache_storage = urlwatcher.cache_storage jobs = [job.with_defaults(urlwatcher.config_storage.config) - for job in urlwatcher.jobs] + for (idx, job) in enumerate(urlwatcher.jobs) if ((idx + 1) in urlwatcher.urlwatch_config.joblist or (not urlwatcher.urlwatch_config.joblist))] report = urlwatcher.report - logger.debug('Processing %d jobs', len(jobs)) + logger.debug('Processing %d jobs (out of %d)', len(jobs), len(urlwatcher.jobs)) with contextlib.ExitStack() as exit_stack: for job_state in run_parallel(lambda job_state: job_state.process(), (exit_stack.enter_context(JobState(cache_storage, job)) for job in jobs)): diff -Nru urlwatch-2.23/README.md urlwatch-2.24/README.md --- urlwatch-2.23/README.md 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/README.md 2021-11-07 08:34:54.000000000 +0000 @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/thp/urlwatch.svg)](https://travis-ci.org/thp/urlwatch) +[![Unit Tests](https://github.com/thp/urlwatch/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/thp/urlwatch/actions/workflows/unit-tests.yml) [![Packaging status](https://repology.org/badge/tiny-repos/urlwatch.svg)](https://repology.org/metapackage/urlwatch/versions) [![PyPI version](https://badge.fury.io/py/urlwatch.svg)](https://badge.fury.io/py/urlwatch) diff -Nru urlwatch-2.23/.travis.yml urlwatch-2.24/.travis.yml --- urlwatch-2.23/.travis.yml 2021-04-10 08:48:22.000000000 +0000 +++ urlwatch-2.24/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -language: python -cache: pip -python: - - "3.6" - - "3.7" - - "3.8" -install: - - pip install -r requirements.txt - - pip install pycodestyle==2.6.0 pytest - - sudo apt-get install -y build-essential libpoppler-cpp-dev pkg-config python-dev tesseract-ocr - - pip install pdftotext docutils pygments pytesseract pillow jq -script: pytest -v