diff -Nru requests-2.9.1/debian/changelog requests-2.10.0/debian/changelog --- requests-2.9.1/debian/changelog 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/changelog 2016-05-23 17:30:48.000000000 +0000 @@ -1,3 +1,23 @@ +requests (2.10.0-1~ubuntu16.10.1~ppa201605231330) yakkety; urgency=medium + + * No-change backport to yakkety + + -- Corey Bryant Mon, 23 May 2016 13:30:48 -0400 + +requests (2.10.0-1) UNRELEASED; urgency=medium + + * New upstream release. + * debian/control + - Bump Standards-Version to 3.9.8 (no changes needed). + - Add python{,3}-socks to Suggests. + - Bump python{,3}-urllib3 dependencies to (>= 1.15.1) (<< 1.15.2). + * debian/copyright + - Update copyright years. + * debian/patches/02_populate-install_requires.patch + - Refresh. + + -- Daniele Tricoli Fri, 20 May 2016 02:41:59 +0200 + requests (2.9.1-3) unstable; urgency=medium * debian/control diff -Nru requests-2.9.1/debian/control requests-2.10.0/debian/control --- requests-2.9.1/debian/control 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/control 2016-05-23 16:35:51.000000000 +0000 @@ -9,12 +9,12 @@ python-all (>= 2.6.6-3), python-chardet, python-setuptools, - python-urllib3 (>= 1.13.1), python-urllib3 (<< 1.13.2), + python-urllib3 (>= 1.15.1), python-urllib3 (<< 1.15.2), python3-all, python3-chardet, python3-setuptools, - python3-urllib3 (>= 1.13.1), python3-urllib3 (<< 1.13.2), -Standards-Version: 3.9.7 + python3-urllib3 (>= 1.15.1), python3-urllib3 (<< 1.15.2), +Standards-Version: 3.9.8 X-Python-Version: >= 2.7 X-Python3-Version: >= 3.3 Homepage: http://python-requests.org @@ -28,11 +28,12 @@ ${python:Depends}, ca-certificates, python-chardet, - python-urllib3 (>= 1.13.1), python-urllib3 (<< 1.13.2) -Suggests: - python-ndg-httpsclient, - python-openssl, - python-pyasn1 + python-urllib3 (>= 1.15.1), python-urllib3 (<< 1.15.2) +Suggests: + python-ndg-httpsclient, + python-openssl, + python-pyasn1, + python-socks Breaks: httpie (<< 0.9.2) Description: elegant and simple HTTP library for Python2, built for human beings Requests allow you to send HTTP/1.1 requests. You can add headers, form data, @@ -60,11 +61,12 @@ ${python3:Depends}, ca-certificates, python3-chardet, - python3-urllib3 (>= 1.13.1), python3-urllib3 (<< 1.13.2) + python3-urllib3 (>= 1.15.1), python3-urllib3 (<< 1.15.2) Suggests: python3-ndg-httpsclient, python3-openssl, - python3-pyasn1 + python3-pyasn1, + python3-socks Description: elegant and simple HTTP library for Python3, built for human beings Requests allow you to send HTTP/1.1 requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access diff -Nru requests-2.9.1/debian/copyright requests-2.10.0/debian/copyright --- requests-2.9.1/debian/copyright 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/copyright 2016-05-23 16:35:51.000000000 +0000 @@ -4,11 +4,11 @@ Source: http://pypi.python.org/pypi/requests Files: * -Copyright: 2015, Kenneth Reitz +Copyright: 2016, Kenneth Reitz License: Apache Files: requests/packages/urllib3/* -Copyright: 2008-2013, Andrey Petrov +Copyright: 2008-2016, Andrey Petrov License: Expat Files: requests/packages/urllib3/packages/ordered_dict.py diff -Nru requests-2.9.1/debian/.git-dpm requests-2.10.0/debian/.git-dpm --- requests-2.9.1/debian/.git-dpm 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/.git-dpm 2016-05-23 16:35:51.000000000 +0000 @@ -1,11 +1,11 @@ # see git-dpm(1) from git-dpm package -fedd3e5275d3388e8d213c592aeca50640ceb372 -fedd3e5275d3388e8d213c592aeca50640ceb372 -8565e660f5f59be865744d0a65a829f61dd16b00 -8565e660f5f59be865744d0a65a829f61dd16b00 -requests_2.9.1.orig.tar.gz -17f01c47a0d7c676f6291608ef2f43db3fa74095 -484252 +490dac0b51a3ba6df874cf91c6be969585573820 +490dac0b51a3ba6df874cf91c6be969585573820 +5ee4efb2658ebe0fad3cc5d12271af49dd073ab3 +5ee4efb2658ebe0fad3cc5d12271af49dd073ab3 +requests_2.10.0.orig.tar.gz +1b9a2395402692262fc61fa15bedbc3e112b1029 +477617 debianTag="debian/%e%v" patchedTag="patched/%e%v" upstreamTag="upstream/%e%u" diff -Nru requests-2.9.1/debian/patches/01_use-system-ca-certificates.patch requests-2.10.0/debian/patches/01_use-system-ca-certificates.patch --- requests-2.9.1/debian/patches/01_use-system-ca-certificates.patch 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/patches/01_use-system-ca-certificates.patch 2016-05-23 16:35:51.000000000 +0000 @@ -1,4 +1,4 @@ -From 34848fcf05a665d46f4ec2e9e86d98a67dcac686 Mon Sep 17 00:00:00 2001 +From b670a11fa2cd604df0351133075be4f8aa1ffb97 Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Thu, 8 Oct 2015 13:41:42 -0700 Subject: Use the bundle provided by ca-certificates instead of @@ -37,10 +37,10 @@ if __name__ == '__main__': print(where()) diff --git a/setup.py b/setup.py -index b7ed12b..c483535 100755 +index 3a39052..3d79616 100755 --- a/setup.py +++ b/setup.py -@@ -50,7 +50,7 @@ setup( +@@ -68,7 +68,7 @@ setup( author_email='me@kennethreitz.com', url='http://python-requests.org', packages=packages, diff -Nru requests-2.9.1/debian/patches/02_populate-install_requires.patch requests-2.10.0/debian/patches/02_populate-install_requires.patch --- requests-2.9.1/debian/patches/02_populate-install_requires.patch 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/patches/02_populate-install_requires.patch 2016-05-23 16:35:51.000000000 +0000 @@ -1,4 +1,4 @@ -From fedd3e5275d3388e8d213c592aeca50640ceb372 Mon Sep 17 00:00:00 2001 +From 490dac0b51a3ba6df874cf91c6be969585573820 Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Fri, 23 Oct 2015 16:03:52 +0200 Subject: Populate install_requires for unbundled packages @@ -15,17 +15,17 @@ 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py -index c483535..2648a3a 100755 +index 3d79616..9a49cdb 100755 --- a/setup.py +++ b/setup.py -@@ -26,7 +26,9 @@ packages = [ +@@ -44,7 +44,9 @@ packages = [ 'requests.packages.urllib3.packages.ssl_match_hostname', ] -requires = [] +requires = [ -+ 'urllib3==1.13.1', ++ 'urllib3==1.15.1', +] + test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov'] - version = '' with open('requests/__init__.py', 'r') as fd: diff -Nru requests-2.9.1/debian/patches/03_export-IncompleteRead.patch requests-2.10.0/debian/patches/03_export-IncompleteRead.patch --- requests-2.9.1/debian/patches/03_export-IncompleteRead.patch 2016-02-12 06:27:18.000000000 +0000 +++ requests-2.10.0/debian/patches/03_export-IncompleteRead.patch 2016-05-23 16:35:51.000000000 +0000 @@ -1,4 +1,4 @@ -From 30dbf485b21229072d097f532566b48521a1a9fa Mon Sep 17 00:00:00 2001 +From 03f38561a6a0efa0e41a09e2d388a920cb035b44 Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Thu, 8 Oct 2015 13:41:43 -0700 Subject: Export IncompleteRead from requests.compat since it's diff -Nru requests-2.9.1/HISTORY.rst requests-2.10.0/HISTORY.rst --- requests-2.9.1/HISTORY.rst 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/HISTORY.rst 2016-04-29 22:11:30.000000000 +0000 @@ -3,6 +3,32 @@ Release History --------------- +2.10.0 (04-29-2016) ++++++++++++++++++++ + +**New Features** + +- SOCKS Proxy Support! (requires PySocks; $ pip install requests[socks]) + +**Miscellaneous** + +- Updated bundled urllib3 to 1.15.1. + +2.9.2 (04-29-2016) +++++++++++++++++++ + +**Improvements** + +- Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict + as its underlying datastore. + +**Bugfixes** + +- Don't use redirect_cache if allow_redirects=False +- When passed objects that throw exceptions from ``tell()``, send them via + chunked transfer encoding instead of failing. +- Raise a ProxyError for proxy related connection issues. + 2.9.1 (2015-12-21) ++++++++++++++++++ diff -Nru requests-2.9.1/LICENSE requests-2.10.0/LICENSE --- requests-2.9.1/LICENSE 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/LICENSE 2016-02-02 07:35:16.000000000 +0000 @@ -1,4 +1,4 @@ -Copyright 2015 Kenneth Reitz +Copyright 2016 Kenneth Reitz Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff -Nru requests-2.9.1/PKG-INFO requests-2.10.0/PKG-INFO --- requests-2.9.1/PKG-INFO 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/PKG-INFO 2016-04-29 22:12:30.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: requests -Version: 2.9.1 +Version: 2.10.0 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -15,41 +15,45 @@ .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests + Requests is the only *Non-GMO* HTTP library for Python, safe for human + consumption. + **Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, + including: security vulnerabilities, verbose code, reinventing the wheel, + constantly reading documentation, depression, headaches, or even death. - - Requests is an Apache2 Licensed HTTP library, written in Python, for human - beings. - - Most existing Python modules for sending HTTP requests are extremely - verbose and cumbersome. Python's builtin urllib2 module provides most of - the HTTP capabilities you should need, but the api is thoroughly broken. - It requires an enormous amount of work (even method overrides) to - perform the simplest of tasks. - - Things shouldn't be this way. Not in Python. + Behold, the power of Requests: .. code-block:: python - >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code - 204 + 200 >>> r.headers['content-type'] - 'application/json' + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' >>> r.text - ... - - See `the same code, without Requests `_. - - Requests allow you to send HTTP/1.1 requests. You can add headers, form data, - multipart files, and parameters with simple Python dictionaries, and access the - response data in the same way. It's powered by httplib and `urllib3 - `_, but it does all the hard work and crazy - hacks for you. + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} + + See `the similar code, sans Requests `_. + + Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the + need for manual labor. There's no need to manually add query strings to your + URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling + are 100% automatic, powered by `urllib3 `_, + which is embedded within Requests. + + Besides, all the cool kids are doing it. Requests is one of the most + downloaded Python packages of all time, pulling in over 7,000,000 downloads + every month. You don't want to be left out! + Feature Support + --------------- - Features - -------- + Requests is ready for today's web. - International Domains and URLs - Keep-Alive & Connection Pooling @@ -58,12 +62,17 @@ - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression + - Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads + - HTTP(S) Proxy Support - Connection Timeouts + - Streaming Downloads + - ``.netrc`` Support + - Chunked Requests - Thread-safety - - HTTP(S) proxy support + Requests supports Python 2.6 — 3.5, and runs great on PyPy. Installation ------------ @@ -73,16 +82,18 @@ .. code-block:: bash $ pip install requests + ✨🍰✨ + Satisfaction, guaranteed. Documentation ------------- - Documentation is available at http://docs.python-requests.org/. + Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. - Contribute - ---------- + How to Contribute + ----------------- #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. #. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). @@ -99,6 +110,32 @@ Release History --------------- + 2.10.0 (04-29-2016) + +++++++++++++++++++ + + **New Features** + + - SOCKS Proxy Support! (requires PySocks; $ pip install requests[socks]) + + **Miscellaneous** + + - Updated bundled urllib3 to 1.15.1. + + 2.9.2 (04-29-2016) + ++++++++++++++++++ + + **Improvements** + + - Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict + as its underlying datastore. + + **Bugfixes** + + - Don't use redirect_cache if allow_redirects=False + - When passed objects that throw exceptions from ``tell()``, send them via + chunked transfer encoding instead of failing. + - Raise a ProxyError for proxy related connection issues. + 2.9.1 (2015-12-21) ++++++++++++++++++ @@ -1231,8 +1268,11 @@ Classifier: Natural Language :: English Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy diff -Nru requests-2.9.1/README.rst requests-2.10.0/README.rst --- requests-2.9.1/README.rst 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/README.rst 2016-02-19 01:41:04.000000000 +0000 @@ -7,41 +7,45 @@ .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests +Requests is the only *Non-GMO* HTTP library for Python, safe for human +consumption. +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +including: security vulnerabilities, verbose code, reinventing the wheel, +constantly reading documentation, depression, headaches, or even death. - -Requests is an Apache2 Licensed HTTP library, written in Python, for human -beings. - -Most existing Python modules for sending HTTP requests are extremely -verbose and cumbersome. Python's builtin urllib2 module provides most of -the HTTP capabilities you should need, but the api is thoroughly broken. -It requires an enormous amount of work (even method overrides) to -perform the simplest of tasks. - -Things shouldn't be this way. Not in Python. +Behold, the power of Requests: .. code-block:: python - >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code - 204 + 200 >>> r.headers['content-type'] - 'application/json' + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' >>> r.text - ... - -See `the same code, without Requests `_. - -Requests allow you to send HTTP/1.1 requests. You can add headers, form data, -multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by httplib and `urllib3 -`_, but it does all the hard work and crazy -hacks for you. + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} + +See `the similar code, sans Requests `_. + +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +need for manual labor. There's no need to manually add query strings to your +URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling +are 100% automatic, powered by `urllib3 `_, +which is embedded within Requests. + +Besides, all the cool kids are doing it. Requests is one of the most +downloaded Python packages of all time, pulling in over 7,000,000 downloads +every month. You don't want to be left out! +Feature Support +--------------- -Features --------- +Requests is ready for today's web. - International Domains and URLs - Keep-Alive & Connection Pooling @@ -50,12 +54,17 @@ - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression +- Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads +- HTTP(S) Proxy Support - Connection Timeouts +- Streaming Downloads +- ``.netrc`` Support +- Chunked Requests - Thread-safety -- HTTP(S) proxy support +Requests supports Python 2.6 — 3.5, and runs great on PyPy. Installation ------------ @@ -65,16 +74,18 @@ .. code-block:: bash $ pip install requests + ✨🍰✨ +Satisfaction, guaranteed. Documentation ------------- -Documentation is available at http://docs.python-requests.org/. +Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. -Contribute ----------- +How to Contribute +----------------- #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. #. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). diff -Nru requests-2.9.1/requests/adapters.py requests-2.10.0/requests/adapters.py --- requests-2.9.1/requests/adapters.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/adapters.py 2016-04-29 22:03:18.000000000 +0000 @@ -19,7 +19,7 @@ from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) + select_proxy, to_native_string) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ClosedPoolError from .packages.urllib3.exceptions import ConnectTimeoutError @@ -33,9 +33,15 @@ from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError) + ProxyError, RetryError, InvalidSchema) from .auth import _basic_auth_str +try: + from .packages.urllib3.contrib.socks import SOCKSProxyManager +except ImportError: + def SOCKSProxyManager(*args, **kwargs): + raise InvalidSchema("Missing dependencies for SOCKS support.") + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -65,7 +71,7 @@ :param pool_connections: The number of urllib3 connection pools to cache. :param pool_maxsize: The maximum number of connections to save in the pool. - :param int max_retries: The maximum number of retries each connection + :param max_retries: The maximum number of retries each connection should attempt. Note, this applies only to failed DNS lookups, socket connections and connection timeouts, never to requests where data has made it to the server. By default, Requests does not retry failed @@ -149,9 +155,22 @@ :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. :returns: ProxyManager """ - if not proxy in self.proxy_manager: + if proxy in self.proxy_manager: + manager = self.proxy_manager[proxy] + elif proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + manager = self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs + ) + else: proxy_headers = self.proxy_headers(proxy) - self.proxy_manager[proxy] = proxy_from_url( + manager = self.proxy_manager[proxy] = proxy_from_url( proxy, proxy_headers=proxy_headers, num_pools=self._pool_connections, @@ -159,7 +178,7 @@ block=self._pool_block, **proxy_kwargs) - return self.proxy_manager[proxy] + return manager def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user @@ -264,10 +283,12 @@ def close(self): """Disposes of any internal state. - Currently, this just closes the PoolManager, which closes pooled - connections. + Currently, this closes the PoolManager and any active ProxyManager, + which closes any pooled connections. """ self.poolmanager.clear() + for proxy in self.proxy_manager.values(): + proxy.clear() def request_url(self, request, proxies): """Obtain the url to use when making the final request. @@ -284,10 +305,16 @@ """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - if proxy and scheme != 'https': + + is_proxied_http_request = (proxy and scheme != 'https') + using_socks_proxy = False + if proxy: + proxy_scheme = urlparse(proxy).scheme.lower() + using_socks_proxy = proxy_scheme.startswith('socks') + + url = request.path_url + if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) - else: - url = request.path_url return url @@ -434,6 +461,9 @@ if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) + if isinstance(e.reason, _ProxyError): + raise ProxyError(e, request=request) + raise ConnectionError(e, request=request) except ClosedPoolError as e: diff -Nru requests-2.9.1/requests/api.py requests-2.10.0/requests/api.py --- requests-2.9.1/requests/api.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/api.py 2016-04-29 21:48:41.000000000 +0000 @@ -24,7 +24,11 @@ :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload. + :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. + ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string + defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers + to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read diff -Nru requests-2.9.1/requests/auth.py requests-2.10.0/requests/auth.py --- requests-2.9.1/requests/auth.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/auth.py 2016-04-29 21:48:41.000000000 +0000 @@ -47,6 +47,15 @@ self.username = username self.password = password + def __eq__(self, other): + return all([ + self.username == getattr(other, 'username', None), + self.password == getattr(other, 'password', None) + ]) + + def __ne__(self, other): + return not self == other + def __call__(self, r): r.headers['Authorization'] = _basic_auth_str(self.username, self.password) return r @@ -84,6 +93,7 @@ qop = self._thread_local.chal.get('qop') algorithm = self._thread_local.chal.get('algorithm') opaque = self._thread_local.chal.get('opaque') + hash_utf8 = None if algorithm is None: _algorithm = 'MD5' @@ -221,3 +231,12 @@ self._thread_local.num_401_calls = 1 return r + + def __eq__(self, other): + return all([ + self.username == getattr(other, 'username', None), + self.password == getattr(other, 'password', None) + ]) + + def __ne__(self, other): + return not self == other diff -Nru requests-2.9.1/requests/cookies.py requests-2.10.0/requests/cookies.py --- requests-2.9.1/requests/cookies.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/cookies.py 2016-04-29 21:48:41.000000000 +0000 @@ -277,6 +277,12 @@ dictionary[cookie.name] = cookie.value return dictionary + def __contains__(self, name): + try: + return super(RequestsCookieJar, self).__contains__(name) + except CookieConflictError: + return True + def __getitem__(self, name): """Dict-like __getitem__() for compatibility with client code. Throws exception if there are more than one cookie with name. In that case, diff -Nru requests-2.9.1/requests/__init__.py requests-2.10.0/requests/__init__.py --- requests-2.9.1/requests/__init__.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/__init__.py 2016-04-29 22:03:19.000000000 +0000 @@ -36,17 +36,17 @@ The other HTTP methods are supported - see `requests.api`. Full documentation is at . -:copyright: (c) 2015 by Kenneth Reitz. +:copyright: (c) 2016 by Kenneth Reitz. :license: Apache 2.0, see LICENSE for more details. """ __title__ = 'requests' -__version__ = '2.9.1' -__build__ = 0x020901 +__version__ = '2.10.0' +__build__ = 0x021000 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2015 Kenneth Reitz' +__copyright__ = 'Copyright 2016 Kenneth Reitz' # Attempt to enable urllib3's SNI support, if possible try: @@ -55,6 +55,12 @@ except ImportError: pass +import warnings + +# urllib3's DependencyWarnings should be silenced. +from .packages.urllib3.exceptions import DependencyWarning +warnings.simplefilter('ignore', DependencyWarning) + from . import utils from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options @@ -63,7 +69,7 @@ from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, + FileModeWarning, ConnectTimeout, ReadTimeout ) # Set default logging handler to avoid "No handler found" warnings. diff -Nru requests-2.9.1/requests/models.py requests-2.10.0/requests/models.py --- requests-2.9.1/requests/models.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/models.py 2016-04-29 21:48:41.000000000 +0000 @@ -103,8 +103,10 @@ """Build the body for a multipart/form-data request. Will successfully encode files when passed as a dict or a list of - 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary + tuples. Order is retained if data is a list of tuples but arbitrary if parameters are supplied as a dict. + The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) + or 4-tuples (filename, fileobj, contentype, custom_headers). """ if (not files): @@ -463,9 +465,11 @@ def prepare_content_length(self, body): if hasattr(body, 'seek') and hasattr(body, 'tell'): + curr_pos = body.tell() body.seek(0, 2) - self.headers['Content-Length'] = builtin_str(body.tell()) - body.seek(0, 0) + end_pos = body.tell() + self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos)) + body.seek(curr_pos, 0) elif body is not None: l = super_len(body) if l: diff -Nru requests-2.9.1/requests/packages/urllib3/_collections.py requests-2.10.0/requests/packages/urllib3/_collections.py --- requests-2.9.1/requests/packages/urllib3/_collections.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/_collections.py 2016-04-06 19:16:56.000000000 +0000 @@ -134,7 +134,7 @@ def __init__(self, headers=None, **kwargs): super(HTTPHeaderDict, self).__init__() - self._container = {} + self._container = OrderedDict() if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) diff -Nru requests-2.9.1/requests/packages/urllib3/connectionpool.py requests-2.10.0/requests/packages/urllib3/connectionpool.py --- requests-2.9.1/requests/packages/urllib3/connectionpool.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/connectionpool.py 2016-04-06 19:16:56.000000000 +0000 @@ -69,7 +69,13 @@ if not host: raise LocationValueError("No host specified.") - self.host = host + # httplib doesn't like it when we include brackets in ipv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. + self.host = host.strip('[]') self.port = port def __str__(self): @@ -203,8 +209,8 @@ Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s" % - (self.num_connections, self.host)) + log.info("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -239,7 +245,7 @@ # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s" % self.host) + log.info("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -272,7 +278,7 @@ except Full: # This should never happen if self.block == True log.warning( - "Connection pool is full, discarding connection: %s" % + "Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. @@ -318,7 +324,7 @@ if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) - def _make_request(self, conn, method, url, timeout=_Default, + def _make_request(self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw): """ Perform a request on a given urllib connection object taken from our @@ -350,7 +356,10 @@ # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. - conn.request(method, url, **httplib_request_kw) + if chunked: + conn.request_chunked(method, url, **httplib_request_kw) + else: + conn.request(method, url, **httplib_request_kw) # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout @@ -382,9 +391,8 @@ # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, - httplib_response.status, - httplib_response.length)) + log.debug("\"%s %s %s\" %s %s", method, url, http_version, + httplib_response.status, httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -435,7 +443,8 @@ def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, **response_kw): + pool_timeout=None, release_conn=None, chunked=False, + **response_kw): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -512,6 +521,11 @@ back into the pool. If None, it takes the value of ``response_kw.get('preload_content', True)``. + :param chunked: + If True, urllib3 will send the body using chunked transfer + encoding. Otherwise, urllib3 will send the body using the standard + content-length form. Defaults to False. + :param \**response_kw: Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` @@ -542,6 +556,10 @@ # complains about UnboundLocalError. err = None + # Keep track of whether we cleanly exited the except block. This + # ensures we do proper cleanup in finally. + clean_exit = False + try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) @@ -556,13 +574,14 @@ # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, timeout=timeout_obj, - body=body, headers=headers) + body=body, headers=headers, + chunked=chunked) # If we're going to release the connection in ``finally:``, then - # the request doesn't need to know about the connection. Otherwise + # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. - response_conn = not release_conn and conn + response_conn = conn if not release_conn else None # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -570,10 +589,8 @@ connection=response_conn, **response_kw) - # else: - # The connection will be put back into the pool when - # ``response.release_conn()`` is called (implicitly by - # ``response.read()``) + # Everything went great! + clean_exit = True except Empty: # Timed out by queue. @@ -583,22 +600,19 @@ # Close the connection. If a connection is reused on which there # was a Certificate error, the next request will certainly raise # another Certificate error. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise SSLError(e) except SSLError: # Treat SSLError separately from BaseSSLError to preserve # traceback. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise except (TimeoutError, HTTPException, SocketError, ProtocolError) as e: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. - conn = conn and conn.close() - release_conn = True + clean_exit = False if isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) @@ -613,6 +627,14 @@ err = e finally: + if not clean_exit: + # We hit some kind of exception, handled or otherwise. We need + # to throw the connection away unless explicitly told not to. + # Close the connection, set the variable to None, and make sure + # we put the None back in the pool to avoid leaking it. + conn = conn and conn.close() + release_conn = True + if release_conn: # Put the connection back to be reused. If the connection is # expired then it will be None, which will get replaced with a @@ -622,7 +644,7 @@ if not conn: # Try again log.warning("Retrying (%r) after connection " - "broken by '%r': %s" % (retries, err, url)) + "broken by '%r': %s", retries, err, url) return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, @@ -644,7 +666,7 @@ raise return response - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -654,9 +676,17 @@ # Check if we should retry the HTTP response. if retries.is_forced_retry(method, status_code=response.status): - retries = retries.increment(method, url, response=response, _pool=self) + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_status: + # Release the connection for this response, since we're not + # returning it to be released manually. + response.release_conn() + raise + return response retries.sleep() - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -742,7 +772,7 @@ except AttributeError: # Platform-specific: Python 2.6 set_tunnel = conn._set_tunnel - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older set_tunnel(self.host, self.port) else: set_tunnel(self.host, self.port, self.proxy_headers) @@ -754,8 +784,8 @@ Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s" - % (self.num_connections, self.host)) + log.info("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -812,6 +842,7 @@ >>> r = conn.request('GET', '/') """ scheme, host, port = get_host(url) + port = port or port_by_scheme.get(scheme, 80) if scheme == 'https': return HTTPSConnectionPool(host, port=port, **kw) else: diff -Nru requests-2.9.1/requests/packages/urllib3/connection.py requests-2.10.0/requests/packages/urllib3/connection.py --- requests-2.9.1/requests/packages/urllib3/connection.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/connection.py 2016-04-06 19:16:56.000000000 +0000 @@ -1,5 +1,6 @@ from __future__ import absolute_import import datetime +import logging import os import sys import socket @@ -38,7 +39,7 @@ SubjectAltNameWarning, SystemTimeWarning, ) -from .packages.ssl_match_hostname import match_hostname +from .packages.ssl_match_hostname import match_hostname, CertificateError from .util.ssl_ import ( resolve_cert_reqs, @@ -50,6 +51,10 @@ from .util import connection +from ._collections import HTTPHeaderDict + +log = logging.getLogger(__name__) + port_by_scheme = { 'http': 80, 'https': 443, @@ -162,6 +167,38 @@ conn = self._new_conn() self._prepare_conn(conn) + def request_chunked(self, method, url, body=None, headers=None): + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + headers = HTTPHeaderDict(headers if headers is not None else {}) + skip_accept_encoding = 'accept-encoding' in headers + self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding) + for header, value in headers.items(): + self.putheader(header, value) + if 'transfer-encoding' not in headers: + self.putheader('Transfer-Encoding', 'chunked') + self.endheaders() + + if body is not None: + stringish_types = six.string_types + (six.binary_type,) + if isinstance(body, stringish_types): + body = (body,) + for chunk in body: + if not chunk: + continue + if not isinstance(chunk, six.binary_type): + chunk = chunk.encode('utf8') + len_str = hex(len(chunk))[2:] + self.send(len_str.encode('utf-8')) + self.send(b'\r\n') + self.send(chunk) + self.send(b'\r\n') + + # After the if clause, to always have a closed body + self.send(b'0\r\n\r\n') + class HTTPSConnection(HTTPConnection): default_port = port_by_scheme['https'] @@ -265,21 +302,26 @@ 'for details.)'.format(hostname)), SubjectAltNameWarning ) - - # In case the hostname is an IPv6 address, strip the square - # brackets from it before using it to validate. This is because - # a certificate with an IPv6 address in it won't have square - # brackets around that address. Sadly, match_hostname won't do this - # for us: it expects the plain host part without any extra work - # that might have been done to make it palatable to httplib. - asserted_hostname = self.assert_hostname or hostname - asserted_hostname = asserted_hostname.strip('[]') - match_hostname(cert, asserted_hostname) + _match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) +def _match_hostname(cert, asserted_hostname): + try: + match_hostname(cert, asserted_hostname) + except CertificateError as e: + log.error( + 'Certificate did not match expected hostname: %s. ' + 'Certificate: %s', asserted_hostname, cert + ) + # Add cert to exception and reraise so client code can inspect + # the cert when catching the exception, if they want to + e._peer_cert = cert + raise + + if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection diff -Nru requests-2.9.1/requests/packages/urllib3/contrib/appengine.py requests-2.10.0/requests/packages/urllib3/contrib/appengine.py --- requests-2.9.1/requests/packages/urllib3/contrib/appengine.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/contrib/appengine.py 2015-12-29 20:28:18.000000000 +0000 @@ -144,7 +144,7 @@ if retries.is_forced_retry(method, status_code=http_response.status): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) retries.sleep() return self.urlopen( method, url, @@ -164,6 +164,14 @@ if content_encoding == 'deflate': del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. @@ -177,7 +185,7 @@ if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if timeout.read is not timeout.connect: + if timeout._read is not timeout._connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) diff -Nru requests-2.9.1/requests/packages/urllib3/contrib/ntlmpool.py requests-2.10.0/requests/packages/urllib3/contrib/ntlmpool.py --- requests-2.9.1/requests/packages/urllib3/contrib/ntlmpool.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/contrib/ntlmpool.py 2015-12-29 20:28:18.000000000 +0000 @@ -43,8 +43,8 @@ # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % - (self.num_connections, self.host, self.authurl)) + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', + self.num_connections, self.host, self.authurl) headers = {} headers['Connection'] = 'Keep-Alive' @@ -56,13 +56,13 @@ # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % reshdr) - log.debug('Response data: %s [...]' % res.read(100)) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', reshdr) + log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) @@ -87,12 +87,12 @@ self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % dict(res.getheaders())) - log.debug('Response data: %s [...]' % res.read()[:100]) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', dict(res.getheaders())) + log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' diff -Nru requests-2.9.1/requests/packages/urllib3/contrib/pyopenssl.py requests-2.10.0/requests/packages/urllib3/contrib/pyopenssl.py --- requests-2.9.1/requests/packages/urllib3/contrib/pyopenssl.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/contrib/pyopenssl.py 2016-04-06 19:16:56.000000000 +0000 @@ -54,9 +54,17 @@ import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint -from socket import _fileobject, timeout, error as SocketError +from socket import timeout, error as SocketError + +try: # Platform-specific: Python 2 + from socket import _fileobject +except ImportError: # Platform-specific: Python 3 + _fileobject = None + from urllib3.packages.backports.makefile import backport_makefile + import ssl import select +import six from .. import connection from .. import util @@ -90,7 +98,7 @@ OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -104,6 +112,7 @@ connection.ssl_wrap_socket = ssl_wrap_socket util.HAS_SNI = HAS_SNI + util.IS_PYOPENSSL = True def extract_from_urllib3(): @@ -111,6 +120,7 @@ connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket util.HAS_SNI = orig_util_HAS_SNI + util.IS_PYOPENSSL = False # Note: This is a slightly bug-fixed version of same from ndg-httpsclient. @@ -135,7 +145,7 @@ for i in range(peer_cert.get_extension_count()): ext = peer_cert.get_extension(i) ext_name = ext.get_short_name() - if ext_name != 'subjectAltName': + if ext_name != b'subjectAltName': continue # PyOpenSSL returns extension data in ASN.1 encoded form @@ -167,13 +177,17 @@ self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs self._makefile_refs = 0 + self._closed = False def fileno(self): return self.socket.fileno() - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) + # Copy-pasted from Python 3.5 source code + def _decref_socketios(self): + if self._makefile_refs > 0: + self._makefile_refs -= 1 + if self._closed: + self.close() def recv(self, *args, **kwargs): try: @@ -182,7 +196,7 @@ if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: - raise SocketError(e) + raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' @@ -198,6 +212,27 @@ else: return data + def recv_into(self, *args, **kwargs): + try: + return self.connection.recv_into(*args, **kwargs) + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return 0 + else: + raise SocketError(str(e)) + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return 0 + else: + raise + except OpenSSL.SSL.WantReadError: + rd, wd, ed = select.select( + [self.socket], [], [], self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + else: + return self.recv_into(*args, **kwargs) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -225,6 +260,7 @@ def close(self): if self._makefile_refs < 1: try: + self._closed = True return self.connection.close() except OpenSSL.SSL.Error: return @@ -262,6 +298,16 @@ self._makefile_refs -= 1 +if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) +else: # Platform-specific: Python 3 + makefile = backport_makefile + +WrappedSocket.makefile = makefile + + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 @@ -285,7 +331,7 @@ else: ctx.set_default_verify_paths() - # Disable TLS compression to migitate CRIME attack (issue #309) + # Disable TLS compression to mitigate CRIME attack (issue #309) OP_NO_COMPRESSION = 0x20000 ctx.set_options(OP_NO_COMPRESSION) @@ -293,6 +339,8 @@ ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST) cnx = OpenSSL.SSL.Connection(ctx, sock) + if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') cnx.set_tlsext_host_name(server_hostname) cnx.set_connect_state() while True: diff -Nru requests-2.9.1/requests/packages/urllib3/contrib/socks.py requests-2.10.0/requests/packages/urllib3/contrib/socks.py --- requests-2.9.1/requests/packages/urllib3/contrib/socks.py 1970-01-01 00:00:00.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/contrib/socks.py 2016-04-06 19:16:56.000000000 +0000 @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +SOCKS support for urllib3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contrib module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from ..exceptions import DependencyWarning + + warnings.warn(( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from ..connection import ( + HTTPConnection, HTTPSConnection +) +from ..connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool +) +from ..exceptions import ConnectTimeoutError, NewConnectionError +from ..poolmanager import PoolManager +from ..util.url import parse_url + +try: + import ssl +except ImportError: + ssl = None + + +class SOCKSConnection(HTTPConnection): + """ + A plain-text HTTP connection that connects via a SOCKS proxy. + """ + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ + Establish a new connection via the SOCKS proxy. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = socks.create_connection( + (self.host, self.port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + timeout=self.timeout, + **extra_kw + ) + + except SocketTimeout as e: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout) + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + + return conn + + +# We don't need to duplicate the Verified/Unverified distinction from +# urllib3/connection.py here because the HTTPSConnection will already have been +# correctly set to either the Verified or Unverified form by that module. This +# means the SOCKSHTTPSConnection will automatically be the correct type. +class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): + pass + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSHTTPSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, + 'https': SOCKSHTTPSConnectionPool, + } + + def __init__(self, proxy_url, username=None, password=None, + num_pools=10, headers=None, **connection_pool_kw): + parsed = parse_url(proxy_url) + + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + } + connection_pool_kw['_socks_options'] = socks_options + + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff -Nru requests-2.9.1/requests/packages/urllib3/exceptions.py requests-2.10.0/requests/packages/urllib3/exceptions.py --- requests-2.9.1/requests/packages/urllib3/exceptions.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/exceptions.py 2015-12-29 20:28:18.000000000 +0000 @@ -180,6 +180,14 @@ pass +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff -Nru requests-2.9.1/requests/packages/urllib3/fields.py requests-2.10.0/requests/packages/urllib3/fields.py --- requests-2.9.1/requests/packages/urllib3/fields.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/fields.py 2016-04-06 19:16:56.000000000 +0000 @@ -36,11 +36,11 @@ result = '%s="%s"' % (name, value) try: result.encode('ascii') - except UnicodeEncodeError: + except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3: # Python 2: + if not six.PY3 and isinstance(value, six.text_type): # Python 2: value = value.encode('utf-8') value = email.utils.encode_rfc2231(value, 'utf-8') value = '%s*=%s' % (name, value) diff -Nru requests-2.9.1/requests/packages/urllib3/__init__.py requests-2.10.0/requests/packages/urllib3/__init__.py --- requests-2.9.1/requests/packages/urllib3/__init__.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/__init__.py 2016-04-11 17:23:23.000000000 +0000 @@ -32,7 +32,7 @@ __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.13.1' +__version__ = '1.15.1' __all__ = ( 'HTTPConnectionPool', @@ -68,22 +68,25 @@ handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s', __name__) return handler # ... Clean up. del NullHandler +# All warning filters *must* be appended unless you're really certain that they +# shouldn't be: otherwise, it's very hard for users to use most Python +# mechanisms to silence them. # SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning) +warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter('default', exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning) +warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): diff -Nru requests-2.9.1/requests/packages/urllib3/poolmanager.py requests-2.10.0/requests/packages/urllib3/poolmanager.py --- requests-2.9.1/requests/packages/urllib3/poolmanager.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/poolmanager.py 2015-12-29 20:28:18.000000000 +0000 @@ -18,16 +18,16 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} - log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + class PoolManager(RequestMethods): """ @@ -65,6 +65,9 @@ self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + # Locally set the pool classes so other PoolManagers can override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + def __enter__(self): return self @@ -81,7 +84,7 @@ by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = pool_classes_by_scheme[scheme] + pool_cls = self.pool_classes_by_scheme[scheme] kwargs = self.connection_pool_kw if scheme == 'http': kwargs = self.connection_pool_kw.copy() @@ -186,7 +189,7 @@ kw['retries'] = retries kw['redirect'] = redirect - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) diff -Nru requests-2.9.1/requests/packages/urllib3/response.py requests-2.10.0/requests/packages/urllib3/response.py --- requests-2.9.1/requests/packages/urllib3/response.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/response.py 2016-04-06 19:16:56.000000000 +0000 @@ -221,6 +221,8 @@ On exit, release the connection back to the pool. """ + clean_exit = False + try: try: yield @@ -243,20 +245,27 @@ # This includes IncompleteRead. raise ProtocolError('Connection broken: %r' % e, e) - except Exception: - # The response may not be closed but we're not going to use it anymore - # so close it now to ensure that the connection is released back to the pool. - if self._original_response and not self._original_response.isclosed(): - self._original_response.close() - - # Closing the response may not actually be sufficient to close - # everything, so if we have a hold of the connection close that - # too. - if self._connection is not None: - self._connection.close() - - raise + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now to ensure that the connection is + # released back to the pool. + if self._original_response: + self._original_response.close() + + # Closing the response may not actually be sufficient to close + # everything, so if we have a hold of the connection close that + # too. + if self._connection: + self._connection.close() + + # If we hold the original response but it's closed now, we should + # return the connection back to the pool. if self._original_response and self._original_response.isclosed(): self.release_conn() @@ -387,6 +396,9 @@ if not self.closed: self._fp.close() + if self._connection: + self._connection.close() + @property def closed(self): if self._fp is None: diff -Nru requests-2.9.1/requests/packages/urllib3/util/__init__.py requests-2.10.0/requests/packages/urllib3/util/__init__.py --- requests-2.9.1/requests/packages/urllib3/util/__init__.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/util/__init__.py 2016-04-06 19:16:56.000000000 +0000 @@ -6,6 +6,7 @@ from .ssl_ import ( SSLContext, HAS_SNI, + IS_PYOPENSSL, assert_fingerprint, resolve_cert_reqs, resolve_ssl_version, @@ -26,6 +27,7 @@ __all__ = ( 'HAS_SNI', + 'IS_PYOPENSSL', 'SSLContext', 'Retry', 'Timeout', diff -Nru requests-2.9.1/requests/packages/urllib3/util/response.py requests-2.10.0/requests/packages/urllib3/util/response.py --- requests-2.9.1/requests/packages/urllib3/util/response.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/util/response.py 2015-12-29 20:28:18.000000000 +0000 @@ -61,7 +61,7 @@ def is_response_to_head(response): """ - Checks, wether a the request of a response has been a HEAD-request. + Checks whether the request of a response has been a HEAD-request. Handles the quirks of AppEngine. :param conn: diff -Nru requests-2.9.1/requests/packages/urllib3/util/retry.py requests-2.10.0/requests/packages/urllib3/util/retry.py --- requests-2.9.1/requests/packages/urllib3/util/retry.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/util/retry.py 2016-04-06 19:16:56.000000000 +0000 @@ -102,6 +102,11 @@ :param bool raise_on_redirect: Whether, if the number of redirects is exhausted, to raise a MaxRetryError, or to return a response with a response code in the 3xx range. + + :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: + whether we should raise an exception, or return a response, + if status falls in ``status_forcelist`` range and retries have + been exhausted. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -112,7 +117,8 @@ def __init__(self, total=10, connect=None, read=None, redirect=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, _observed_errors=0): + backoff_factor=0, raise_on_redirect=True, raise_on_status=True, + _observed_errors=0): self.total = total self.connect = connect @@ -127,6 +133,7 @@ self.method_whitelist = method_whitelist self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect + self.raise_on_status = raise_on_status self._observed_errors = _observed_errors # TODO: use .history instead? def new(self, **kw): @@ -137,6 +144,7 @@ status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, _observed_errors=self._observed_errors, ) params.update(kw) @@ -153,7 +161,7 @@ redirect = bool(redirect) and None new_retries = cls(retries, redirect=redirect) - log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries def get_backoff_time(self): @@ -272,7 +280,7 @@ if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) - log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry diff -Nru requests-2.9.1/requests/packages/urllib3/util/ssl_.py requests-2.10.0/requests/packages/urllib3/util/ssl_.py --- requests-2.9.1/requests/packages/urllib3/util/ssl_.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/packages/urllib3/util/ssl_.py 2016-04-06 19:16:56.000000000 +0000 @@ -12,6 +12,7 @@ SSLContext = None HAS_SNI = False create_default_context = None +IS_PYOPENSSL = False # Maps the length of a digest to a possible hash function producing this digest HASHFUNC_MAP = { @@ -110,11 +111,12 @@ ) self.ciphers = cipher_suite - def wrap_socket(self, socket, server_hostname=None): + def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. For more information, see ' + 'certain SSL connections to fail. You can upgrade to a newer ' + 'version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#insecureplatformwarning.', InsecurePlatformWarning @@ -125,6 +127,7 @@ 'ca_certs': self.ca_certs, 'cert_reqs': self.verify_mode, 'ssl_version': self.protocol, + 'server_side': server_side, } if self.supports_set_ciphers: # Platform-specific: Python 2.7+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs) @@ -308,8 +311,8 @@ 'An HTTPS request has been made, but the SNI (Subject Name ' 'Indication) extension to TLS is not available on this platform. ' 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. For more ' - 'information, see ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#snimissingwarning.', SNIMissingWarning diff -Nru requests-2.9.1/requests/sessions.py requests-2.10.0/requests/sessions.py --- requests-2.9.1/requests/sessions.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/sessions.py 2016-04-29 21:48:41.000000000 +0000 @@ -110,13 +110,12 @@ resp.raw.read(decode_content=False) if i >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects) + raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) # Release the connection back into the pool. resp.close() url = resp.headers['location'] - method = req.method # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -140,22 +139,7 @@ if resp.is_permanent_redirect and req.url != prepared_request.url: self.redirect_cache[req.url] = prepared_request.url - # http://tools.ietf.org/html/rfc7231#section-6.4.4 - if (resp.status_code == codes.see_other and - method != 'HEAD'): - method = 'GET' - - # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if resp.status_code == codes.found and method != 'HEAD': - method = 'GET' - - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if resp.status_code == codes.moved and method == 'POST': - method = 'GET' - - prepared_request.method = method + self.rebuild_method(prepared_request, resp) # https://github.com/kennethreitz/requests/issues/1084 if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): @@ -262,6 +246,28 @@ return new_proxies + def rebuild_method(self, prepared_request, response): + """When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = prepared_request.method + + # http://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.see_other and method != 'HEAD': + method = 'GET' + + # Do what the browsers do, despite standards... + # First, turn 302s into GETs. + if response.status_code == codes.found and method != 'HEAD': + method = 'GET' + + # Second, if a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in Issue 1704. + if response.status_code == codes.moved and method == 'POST': + method = 'GET' + + prepared_request.method = method + class Session(SessionRedirectMixin): """A Requests session. @@ -437,7 +443,8 @@ A CA_BUNDLE path can also be provided. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. - """ + :rtype: requests.Response + """ # Create the Request. req = Request( method = method.upper(), @@ -550,22 +557,24 @@ # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. - if not isinstance(request, PreparedRequest): + if isinstance(request, Request): raise ValueError('You can only send PreparedRequests.') - checked_urls = set() - while request.url in self.redirect_cache: - checked_urls.add(request.url) - new_url = self.redirect_cache.get(request.url) - if new_url in checked_urls: - break - request.url = new_url - # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks + # Resolve URL in redirect cache, if available. + if allow_redirects: + checked_urls = set() + while request.url in self.redirect_cache: + checked_urls.add(request.url) + new_url = self.redirect_cache.get(request.url) + if new_url in checked_urls: + break + request.url = new_url + # Get the appropriate adapter to use adapter = self.get_adapter(url=request.url) diff -Nru requests-2.9.1/requests/status_codes.py requests-2.10.0/requests/status_codes.py --- requests-2.9.1/requests/status_codes.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/status_codes.py 2016-04-29 21:48:41.000000000 +0000 @@ -53,6 +53,7 @@ 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 417: ('expectation_failed',), 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 421: ('misdirected_request',), 422: ('unprocessable_entity', 'unprocessable'), 423: ('locked',), 424: ('failed_dependency', 'dependency'), diff -Nru requests-2.9.1/requests/structures.py requests-2.10.0/requests/structures.py --- requests-2.9.1/requests/structures.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/structures.py 2016-04-29 21:48:41.000000000 +0000 @@ -10,6 +10,8 @@ import collections +from .compat import OrderedDict + class CaseInsensitiveDict(collections.MutableMapping): """ @@ -40,7 +42,7 @@ """ def __init__(self, data=None, **kwargs): - self._store = dict() + self._store = OrderedDict() if data is None: data = {} self.update(data, **kwargs) diff -Nru requests-2.9.1/requests/utils.py requests-2.10.0/requests/utils.py --- requests-2.9.1/requests/utils.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests/utils.py 2016-04-29 21:48:41.000000000 +0000 @@ -14,9 +14,7 @@ import collections import io import os -import platform import re -import sys import socket import struct import warnings @@ -83,7 +81,14 @@ ) if hasattr(o, 'tell'): - current_position = o.tell() + try: + current_position = o.tell() + except (OSError, IOError): + # This can happen in some weird situations, such as when the file + # is actually a special file descriptor like stdin. In this + # instance, we don't know what the length is, so set it to zero and + # let requests chunk it instead. + current_position = total_length return max(0, total_length - current_position) @@ -557,6 +562,7 @@ return False + def get_environ_proxies(url): """Return a dict of environment proxies.""" if should_bypass_proxies(url): @@ -564,6 +570,7 @@ else: return getproxies() + def select_proxy(url, proxies): """Select a proxy for the url, if applicable. @@ -572,11 +579,15 @@ """ proxies = proxies or {} urlparts = urlparse(url) - proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) + if urlparts.hostname is None: + proxy = None + else: + proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) if proxy is None: proxy = proxies.get(urlparts.scheme) return proxy + def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" return '%s/%s' % (name, __version__) @@ -600,21 +611,19 @@ links = [] - replace_chars = " '\"" + replace_chars = ' \'"' - for val in re.split(", *<", value): + for val in re.split(', *<', value): try: - url, params = val.split(";", 1) + url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {} - - link["url"] = url.strip("<> '\"") + link = {'url': url.strip('<> \'"')} - for param in params.split(";"): + for param in params.split(';'): try: - key, value = param.split("=") + key, value = param.split('=') except ValueError: break @@ -661,8 +670,8 @@ def prepend_scheme_if_needed(url, new_scheme): - '''Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.''' + """Given a URL that may or may not have a scheme, prepend the given scheme. + Does not replace a present scheme with the one provided as an argument.""" scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -693,8 +702,6 @@ string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. """ - out = None - if isinstance(string, builtin_str): out = string else: diff -Nru requests-2.9.1/requests.egg-info/PKG-INFO requests-2.10.0/requests.egg-info/PKG-INFO --- requests-2.9.1/requests.egg-info/PKG-INFO 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests.egg-info/PKG-INFO 2016-04-29 22:12:30.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: requests -Version: 2.9.1 +Version: 2.10.0 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz @@ -15,41 +15,45 @@ .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests + Requests is the only *Non-GMO* HTTP library for Python, safe for human + consumption. + **Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, + including: security vulnerabilities, verbose code, reinventing the wheel, + constantly reading documentation, depression, headaches, or even death. - - Requests is an Apache2 Licensed HTTP library, written in Python, for human - beings. - - Most existing Python modules for sending HTTP requests are extremely - verbose and cumbersome. Python's builtin urllib2 module provides most of - the HTTP capabilities you should need, but the api is thoroughly broken. - It requires an enormous amount of work (even method overrides) to - perform the simplest of tasks. - - Things shouldn't be this way. Not in Python. + Behold, the power of Requests: .. code-block:: python - >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code - 204 + 200 >>> r.headers['content-type'] - 'application/json' + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' >>> r.text - ... - - See `the same code, without Requests `_. - - Requests allow you to send HTTP/1.1 requests. You can add headers, form data, - multipart files, and parameters with simple Python dictionaries, and access the - response data in the same way. It's powered by httplib and `urllib3 - `_, but it does all the hard work and crazy - hacks for you. + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} + + See `the similar code, sans Requests `_. + + Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the + need for manual labor. There's no need to manually add query strings to your + URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling + are 100% automatic, powered by `urllib3 `_, + which is embedded within Requests. + + Besides, all the cool kids are doing it. Requests is one of the most + downloaded Python packages of all time, pulling in over 7,000,000 downloads + every month. You don't want to be left out! + Feature Support + --------------- - Features - -------- + Requests is ready for today's web. - International Domains and URLs - Keep-Alive & Connection Pooling @@ -58,12 +62,17 @@ - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression + - Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads + - HTTP(S) Proxy Support - Connection Timeouts + - Streaming Downloads + - ``.netrc`` Support + - Chunked Requests - Thread-safety - - HTTP(S) proxy support + Requests supports Python 2.6 — 3.5, and runs great on PyPy. Installation ------------ @@ -73,16 +82,18 @@ .. code-block:: bash $ pip install requests + ✨🍰✨ + Satisfaction, guaranteed. Documentation ------------- - Documentation is available at http://docs.python-requests.org/. + Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. - Contribute - ---------- + How to Contribute + ----------------- #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. #. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). @@ -99,6 +110,32 @@ Release History --------------- + 2.10.0 (04-29-2016) + +++++++++++++++++++ + + **New Features** + + - SOCKS Proxy Support! (requires PySocks; $ pip install requests[socks]) + + **Miscellaneous** + + - Updated bundled urllib3 to 1.15.1. + + 2.9.2 (04-29-2016) + ++++++++++++++++++ + + **Improvements** + + - Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict + as its underlying datastore. + + **Bugfixes** + + - Don't use redirect_cache if allow_redirects=False + - When passed objects that throw exceptions from ``tell()``, send them via + chunked transfer encoding instead of failing. + - Raise a ProxyError for proxy related connection issues. + 2.9.1 (2015-12-21) ++++++++++++++++++ @@ -1231,8 +1268,11 @@ Classifier: Natural Language :: English Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy diff -Nru requests-2.9.1/requests.egg-info/requires.txt requests-2.10.0/requests.egg-info/requires.txt --- requests-2.9.1/requests.egg-info/requires.txt 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests.egg-info/requires.txt 2016-04-29 22:12:30.000000000 +0000 @@ -3,3 +3,6 @@ pyOpenSSL>=0.13 ndg-httpsclient pyasn1 + +[socks] +PySocks>=1.5.6 diff -Nru requests-2.9.1/requests.egg-info/SOURCES.txt requests-2.10.0/requests.egg-info/SOURCES.txt --- requests-2.9.1/requests.egg-info/SOURCES.txt 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requests.egg-info/SOURCES.txt 2016-04-29 22:12:30.000000000 +0000 @@ -4,9 +4,7 @@ NOTICE README.rst requirements.txt -setup.cfg setup.py -test_requests.py requests/__init__.py requests/adapters.py requests/api.py @@ -81,6 +79,7 @@ requests/packages/urllib3/contrib/appengine.py requests/packages/urllib3/contrib/ntlmpool.py requests/packages/urllib3/contrib/pyopenssl.py +requests/packages/urllib3/contrib/socks.py requests/packages/urllib3/packages/__init__.py requests/packages/urllib3/packages/ordered_dict.py requests/packages/urllib3/packages/six.py diff -Nru requests-2.9.1/requirements.txt requests-2.10.0/requirements.txt --- requests-2.9.1/requirements.txt 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/requirements.txt 2016-04-29 21:48:41.000000000 +0000 @@ -1,6 +1,23 @@ -py==1.4.30 -pytest==2.8.1 -pytest-cov==2.1.0 -pytest-httpbin==0.0.7 -httpbin==0.4.0 -wheel +alabaster==0.7.7 +Babel==2.2.0 +coverage==4.0.3 +decorator==4.0.9 +docutils==0.12 +Flask==0.10.1 +httpbin==0.4.1 +itsdangerous==0.24 +Jinja2==2.8 +MarkupSafe==0.23 +py==1.4.31 +Pygments==2.1.1 +pytest==2.8.7 +pytest-cov==2.2.1 +pytest-httpbin==0.2.0 +pytest-mock==0.11.0 +pytz==2015.7 +six==1.10.0 +snowballstemmer==1.2.1 +Sphinx==1.3.5 +sphinx-rtd-theme==0.1.9 +Werkzeug==0.11.4 +wheel==0.29.0 diff -Nru requests-2.9.1/setup.cfg requests-2.10.0/setup.cfg --- requests-2.9.1/setup.cfg 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/setup.cfg 2016-04-29 22:12:30.000000000 +0000 @@ -1,6 +1,3 @@ -[wheel] -universal = 1 - [egg_info] tag_build = tag_date = 0 diff -Nru requests-2.9.1/setup.py requests-2.10.0/setup.py --- requests-2.9.1/setup.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/setup.py 2016-04-29 22:03:18.000000000 +0000 @@ -6,10 +6,28 @@ from codecs import open -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = [] + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + + errno = pytest.main(self.pytest_args) + sys.exit(errno) + if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') @@ -27,8 +45,8 @@ ] requires = [] +test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov'] -version = '' with open('requests/__init__.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) @@ -62,13 +80,19 @@ 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy' ), + cmdclass={'test': PyTest}, + tests_require=test_requirements, extras_require={ 'security': ['pyOpenSSL>=0.13', 'ndg-httpsclient', 'pyasn1'], + 'socks': ['PySocks>=1.5.6'], }, ) diff -Nru requests-2.9.1/test_requests.py requests-2.10.0/test_requests.py --- requests-2.9.1/test_requests.py 2015-12-27 09:59:26.000000000 +0000 +++ requests-2.10.0/test_requests.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,1746 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""Tests for Requests.""" - -from __future__ import division -import json -import os -import pickle -import unittest -import collections -import contextlib - -import io -import requests -import pytest -from requests.adapters import HTTPAdapter -from requests.auth import HTTPDigestAuth, _basic_auth_str -from requests.compat import ( - Morsel, cookielib, getproxies, str, urljoin, urlparse, is_py3, - builtin_str, OrderedDict - ) -from requests.cookies import cookiejar_from_dict, morsel_to_cookie -from requests.exceptions import (ConnectionError, ConnectTimeout, - InvalidSchema, InvalidURL, MissingSchema, - ReadTimeout, Timeout, RetryError) -from requests.models import PreparedRequest -from requests.structures import CaseInsensitiveDict -from requests.sessions import SessionRedirectMixin -from requests.models import urlencode -from requests.hooks import default_hooks - -try: - import StringIO -except ImportError: - import io as StringIO - -try: - from multiprocessing.pool import ThreadPool -except ImportError: - ThreadPool = None - -if is_py3: - def u(s): - return s -else: - def u(s): - return s.decode('unicode-escape') - - -@pytest.fixture -def httpbin(httpbin): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - - -@pytest.fixture -def httpsbin_url(httpbin_secure): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin_secure.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - - -# Requests to this URL should always fail with a connection timeout (nothing -# listening on that port) -TARPIT = "http://10.255.255.1" - -class TestRequests(object): - - _multiprocess_can_split_ = True - - def setUp(self): - """Create simple data set with headers.""" - pass - - def tearDown(self): - """Teardown.""" - pass - - def test_entry_points(self): - - requests.session - requests.session().get - requests.session().head - requests.get - requests.head - requests.put - requests.patch - requests.post - - def test_invalid_url(self): - with pytest.raises(MissingSchema): - requests.get('hiwpefhipowhefopw') - with pytest.raises(InvalidSchema): - requests.get('localhost:3128') - with pytest.raises(InvalidSchema): - requests.get('localhost.localdomain:3128/') - with pytest.raises(InvalidSchema): - requests.get('10.122.1.1:3128/') - with pytest.raises(InvalidURL): - requests.get('http://') - - def test_basic_building(self): - req = requests.Request() - req.url = 'http://kennethreitz.org/' - req.data = {'life': '42'} - - pr = req.prepare() - assert pr.url == req.url - assert pr.body == 'life=42' - - def test_no_content_length(self, httpbin): - get_req = requests.Request('GET', httpbin('get')).prepare() - assert 'Content-Length' not in get_req.headers - head_req = requests.Request('HEAD', httpbin('head')).prepare() - assert 'Content-Length' not in head_req.headers - - def test_override_content_length(self, httpbin): - headers = { - 'Content-Length': 'not zero' - } - r = requests.Request('POST', httpbin('post'), headers=headers).prepare() - assert 'Content-Length' in r.headers - assert r.headers['Content-Length'] == 'not zero' - - def test_path_is_not_double_encoded(self): - request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare() - - assert request.path_url == '/get/test%20case' - - def test_params_are_added_before_fragment(self): - request = requests.Request('GET', - "http://example.com/path#fragment", params={"a": "b"}).prepare() - assert request.url == "http://example.com/path?a=b#fragment" - request = requests.Request('GET', - "http://example.com/path?key=value#fragment", params={"a": "b"}).prepare() - assert request.url == "http://example.com/path?key=value&a=b#fragment" - - def test_params_original_order_is_preserved_by_default(self): - param_ordered_dict = OrderedDict((('z', 1), ('a', 1), ('k', 1), ('d', 1))) - session = requests.Session() - request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict) - prep = session.prepare_request(request) - assert prep.url == 'http://example.com/?z=1&a=1&k=1&d=1' - - def test_params_bytes_are_encoded(self): - request = requests.Request('GET', 'http://example.com', - params=b'test=foo').prepare() - assert request.url == 'http://example.com/?test=foo' - - def test_binary_put(self): - request = requests.Request('PUT', 'http://example.com', - data=u"ööö".encode("utf-8")).prepare() - assert isinstance(request.body, bytes) - - def test_mixed_case_scheme_acceptable(self, httpbin): - s = requests.Session() - s.proxies = getproxies() - parts = urlparse(httpbin('get')) - schemes = ['http://', 'HTTP://', 'hTTp://', 'HttP://'] - for scheme in schemes: - url = scheme + parts.netloc + parts.path - r = requests.Request('GET', url) - r = s.send(r.prepare()) - assert r.status_code == 200, 'failed for scheme {0}'.format(scheme) - - def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin): - r = requests.Request('GET', httpbin('get')) - s = requests.Session() - s.proxies = getproxies() - - r = s.send(r.prepare()) - - assert r.status_code == 200 - - def test_HTTP_302_ALLOW_REDIRECT_GET(self, httpbin): - r = requests.get(httpbin('redirect', '1')) - assert r.status_code == 200 - assert r.history[0].status_code == 302 - assert r.history[0].is_redirect - - # def test_HTTP_302_ALLOW_REDIRECT_POST(self): - # r = requests.post(httpbin('status', '302'), data={'some': 'data'}) - # self.assertEqual(r.status_code, 200) - - def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): - heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('user-agent'), headers=heads) - - assert heads['User-agent'] in r.text - assert r.status_code == 200 - - def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self, httpbin): - heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) - assert r.status_code == 200 - - def test_set_cookie_on_301(self, httpbin): - s = requests.session() - url = httpbin('cookies/set?foo=bar') - s.get(url) - assert s.cookies['foo'] == 'bar' - - def test_cookie_sent_on_redirect(self, httpbin): - s = requests.session() - s.get(httpbin('cookies/set?foo=bar')) - r = s.get(httpbin('redirect/1')) # redirects to httpbin('get') - assert 'Cookie' in r.json()['headers'] - - def test_cookie_removed_on_expire(self, httpbin): - s = requests.session() - s.get(httpbin('cookies/set?foo=bar')) - assert s.cookies['foo'] == 'bar' - s.get( - httpbin('response-headers'), - params={ - 'Set-Cookie': - 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT' - } - ) - assert 'foo' not in s.cookies - - def test_cookie_quote_wrapped(self, httpbin): - s = requests.session() - s.get(httpbin('cookies/set?foo="bar:baz"')) - assert s.cookies['foo'] == '"bar:baz"' - - def test_cookie_persists_via_api(self, httpbin): - s = requests.session() - r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'}) - assert 'foo' in r.request.headers['Cookie'] - assert 'foo' in r.history[0].request.headers['Cookie'] - - def test_request_cookie_overrides_session_cookie(self, httpbin): - s = requests.session() - s.cookies['foo'] = 'bar' - r = s.get(httpbin('cookies'), cookies={'foo': 'baz'}) - assert r.json()['cookies']['foo'] == 'baz' - # Session cookie should not be modified - assert s.cookies['foo'] == 'bar' - - def test_request_cookies_not_persisted(self, httpbin): - s = requests.session() - s.get(httpbin('cookies'), cookies={'foo': 'baz'}) - # Sending a request with cookies should not add cookies to the session - assert not s.cookies - - def test_generic_cookiejar_works(self, httpbin): - cj = cookielib.CookieJar() - cookiejar_from_dict({'foo': 'bar'}, cj) - s = requests.session() - s.cookies = cj - r = s.get(httpbin('cookies')) - # Make sure the cookie was sent - assert r.json()['cookies']['foo'] == 'bar' - # Make sure the session cj is still the custom one - assert s.cookies is cj - - def test_param_cookiejar_works(self, httpbin): - cj = cookielib.CookieJar() - cookiejar_from_dict({'foo': 'bar'}, cj) - s = requests.session() - r = s.get(httpbin('cookies'), cookies=cj) - # Make sure the cookie was sent - assert r.json()['cookies']['foo'] == 'bar' - - def test_requests_in_history_are_not_overridden(self, httpbin): - resp = requests.get(httpbin('redirect/3')) - urls = [r.url for r in resp.history] - req_urls = [r.request.url for r in resp.history] - assert urls == req_urls - - def test_history_is_always_a_list(self, httpbin): - """ - Show that even with redirects, Response.history is always a list. - """ - resp = requests.get(httpbin('get')) - assert isinstance(resp.history, list) - resp = requests.get(httpbin('redirect/1')) - assert isinstance(resp.history, list) - assert not isinstance(resp.history, tuple) - - def test_headers_on_session_with_None_are_not_sent(self, httpbin): - """Do not send headers in Session.headers with None values.""" - ses = requests.Session() - ses.headers['Accept-Encoding'] = None - req = requests.Request('GET', httpbin('get')) - prep = ses.prepare_request(req) - assert 'Accept-Encoding' not in prep.headers - - def test_user_agent_transfers(self, httpbin): - - heads = { - 'User-agent': 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } - - r = requests.get(httpbin('user-agent'), headers=heads) - assert heads['User-agent'] in r.text - - heads = { - 'user-agent': 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } - - r = requests.get(httpbin('user-agent'), headers=heads) - assert heads['user-agent'] in r.text - - def test_HTTP_200_OK_HEAD(self, httpbin): - r = requests.head(httpbin('get')) - assert r.status_code == 200 - - def test_HTTP_200_OK_PUT(self, httpbin): - r = requests.put(httpbin('put')) - assert r.status_code == 200 - - def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin): - auth = ('user', 'pass') - url = httpbin('basic-auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert r.status_code == 200 - - r = requests.get(url) - assert r.status_code == 401 - - s = requests.session() - s.auth = auth - r = s.get(url) - assert r.status_code == 200 - - def test_connection_error_invalid_domain(self): - """Connecting to an unknown domain should raise a ConnectionError""" - with pytest.raises(ConnectionError): - requests.get("http://doesnotexist.google.com") - - def test_connection_error_invalid_port(self): - """Connecting to an invalid port should raise a ConnectionError""" - with pytest.raises(ConnectionError): - requests.get("http://localhost:1", timeout=1) - - def test_LocationParseError(self): - """Inputing a URL that cannot be parsed should raise an InvalidURL error""" - with pytest.raises(InvalidURL): - requests.get("http://fe80::5054:ff:fe5a:fc0") - - def test_basicauth_with_netrc(self, httpbin): - auth = ('user', 'pass') - wrong_auth = ('wronguser', 'wrongpass') - url = httpbin('basic-auth', 'user', 'pass') - - old_auth = requests.sessions.get_netrc_auth - - try: - def get_netrc_auth_mock(url): - return auth - requests.sessions.get_netrc_auth = get_netrc_auth_mock - - # Should use netrc and work. - r = requests.get(url) - assert r.status_code == 200 - - # Given auth should override and fail. - r = requests.get(url, auth=wrong_auth) - assert r.status_code == 401 - - s = requests.session() - - # Should use netrc and work. - r = s.get(url) - assert r.status_code == 200 - - # Given auth should override and fail. - s.auth = wrong_auth - r = s.get(url) - assert r.status_code == 401 - finally: - requests.sessions.get_netrc_auth = old_auth - - def test_DIGEST_HTTP_200_OK_GET(self, httpbin): - - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert r.status_code == 200 - - r = requests.get(url) - assert r.status_code == 401 - - s = requests.session() - s.auth = HTTPDigestAuth('user', 'pass') - r = s.get(url) - assert r.status_code == 200 - - def test_DIGEST_AUTH_RETURNS_COOKIE(self, httpbin): - url = httpbin('digest-auth', 'auth', 'user', 'pass') - auth = HTTPDigestAuth('user', 'pass') - r = requests.get(url) - assert r.cookies['fake'] == 'fake_value' - - r = requests.get(url, auth=auth) - assert r.status_code == 200 - - def test_DIGEST_AUTH_SETS_SESSION_COOKIES(self, httpbin): - url = httpbin('digest-auth', 'auth', 'user', 'pass') - auth = HTTPDigestAuth('user', 'pass') - s = requests.Session() - s.get(url, auth=auth) - assert s.cookies['fake'] == 'fake_value' - - def test_DIGEST_STREAM(self, httpbin): - - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth, stream=True) - assert r.raw.read() != b'' - - r = requests.get(url, auth=auth, stream=False) - assert r.raw.read() == b'' - - def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin): - - auth = HTTPDigestAuth('user', 'wrongpass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert r.status_code == 401 - - r = requests.get(url) - assert r.status_code == 401 - - s = requests.session() - s.auth = auth - r = s.get(url) - assert r.status_code == 401 - - def test_DIGESTAUTH_QUOTES_QOP_VALUE(self, httpbin): - - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - assert '"auth"' in r.request.headers['Authorization'] - - def test_POSTBIN_GET_POST_FILES(self, httpbin): - - url = httpbin('post') - post1 = requests.post(url).raise_for_status() - - post1 = requests.post(url, data={'some': 'data'}) - assert post1.status_code == 200 - - with open('requirements.txt') as f: - post2 = requests.post(url, files={'some': f}) - assert post2.status_code == 200 - - post4 = requests.post(url, data='[{"some": "json"}]') - assert post4.status_code == 200 - - with pytest.raises(ValueError): - requests.post(url, files=['bad file data']) - - def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin): - - url = httpbin('post') - post1 = requests.post(url).raise_for_status() - - post1 = requests.post(url, data={'some': 'data'}) - assert post1.status_code == 200 - - with open('requirements.txt') as f: - post2 = requests.post(url, - data={'some': 'data'}, files={'some': f}) - assert post2.status_code == 200 - - post4 = requests.post(url, data='[{"some": "json"}]') - assert post4.status_code == 200 - - with pytest.raises(ValueError): - requests.post(url, files=['bad file data']) - - def test_conflicting_post_params(self, httpbin): - url = httpbin('post') - with open('requirements.txt') as f: - pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})") - pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})") - - def test_request_ok_set(self, httpbin): - r = requests.get(httpbin('status', '404')) - assert not r.ok - - def test_status_raising(self, httpbin): - r = requests.get(httpbin('status', '404')) - with pytest.raises(requests.exceptions.HTTPError): - r.raise_for_status() - - r = requests.get(httpbin('status', '500')) - assert not r.ok - - def test_decompress_gzip(self, httpbin): - r = requests.get(httpbin('gzip')) - r.content.decode('ascii') - - def test_unicode_get(self, httpbin): - url = httpbin('/get') - requests.get(url, params={'foo': 'føø'}) - requests.get(url, params={'føø': 'føø'}) - requests.get(url, params={'føø': 'føø'}) - requests.get(url, params={'foo': 'foo'}) - requests.get(httpbin('ø'), params={'foo': 'foo'}) - - def test_unicode_header_name(self, httpbin): - requests.put( - httpbin('put'), - headers={str('Content-Type'): 'application/octet-stream'}, - data='\xff') # compat.str is unicode. - - def test_pyopenssl_redirect(self, httpsbin_url, httpbin_ca_bundle): - requests.get(httpsbin_url('status', '301'), verify=httpbin_ca_bundle) - - def test_urlencoded_get_query_multivalued_param(self, httpbin): - - r = requests.get(httpbin('get'), params=dict(test=['foo', 'baz'])) - assert r.status_code == 200 - assert r.url == httpbin('get?test=foo&test=baz') - - def test_different_encodings_dont_break_post(self, httpbin): - r = requests.post(httpbin('post'), - data={'stuff': json.dumps({'a': 123})}, - params={'blah': 'asdf1234'}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) - assert r.status_code == 200 - - def test_unicode_multipart_post(self, httpbin): - r = requests.post(httpbin('post'), - data={'stuff': u('ëlïxr')}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) - assert r.status_code == 200 - - r = requests.post(httpbin('post'), - data={'stuff': u('ëlïxr').encode('utf-8')}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) - assert r.status_code == 200 - - r = requests.post(httpbin('post'), - data={'stuff': 'elixr'}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) - assert r.status_code == 200 - - r = requests.post(httpbin('post'), - data={'stuff': 'elixr'.encode('utf-8')}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) - assert r.status_code == 200 - - def test_unicode_multipart_post_fieldnames(self, httpbin): - filename = os.path.splitext(__file__)[0] + '.py' - r = requests.Request(method='POST', - url=httpbin('post'), - data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', - open(filename, 'rb'))}) - prep = r.prepare() - assert b'name="stuff"' in prep.body - assert b'name="b\'stuff\'"' not in prep.body - - def test_unicode_method_name(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} - r = requests.request( - method=u('POST'), url=httpbin('post'), files=files) - assert r.status_code == 200 - - def test_unicode_method_name_with_request_object(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} - s = requests.Session() - req = requests.Request(u("POST"), httpbin('post'), files=files) - prep = s.prepare_request(req) - assert isinstance(prep.method, builtin_str) - assert prep.method == "POST" - - resp = s.send(prep) - assert resp.status_code == 200 - - def test_custom_content_type(self, httpbin): - r = requests.post( - httpbin('post'), - data={'stuff': json.dumps({'a': 123})}, - files={'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) - assert r.status_code == 200 - assert b"text/py-content-type" in r.request.body - - def test_hook_receives_request_arguments(self, httpbin): - def hook(resp, **kwargs): - assert resp is not None - assert kwargs != {} - - requests.Request('GET', httpbin(), hooks={'response': hook}) - - def test_session_hooks_are_used_with_no_request_hooks(self, httpbin): - hook = lambda x, *args, **kwargs: x - s = requests.Session() - s.hooks['response'].append(hook) - r = requests.Request('GET', httpbin()) - prep = s.prepare_request(r) - assert prep.hooks['response'] != [] - assert prep.hooks['response'] == [hook] - - def test_session_hooks_are_overridden_by_request_hooks(self, httpbin): - hook1 = lambda x, *args, **kwargs: x - hook2 = lambda x, *args, **kwargs: x - assert hook1 is not hook2 - s = requests.Session() - s.hooks['response'].append(hook2) - r = requests.Request('GET', httpbin(), hooks={'response': [hook1]}) - prep = s.prepare_request(r) - assert prep.hooks['response'] == [hook1] - - def test_prepared_request_hook(self, httpbin): - def hook(resp, **kwargs): - resp.hook_working = True - return resp - - req = requests.Request('GET', httpbin(), hooks={'response': hook}) - prep = req.prepare() - - s = requests.Session() - s.proxies = getproxies() - resp = s.send(prep) - - assert hasattr(resp, 'hook_working') - - def test_prepared_from_session(self, httpbin): - class DummyAuth(requests.auth.AuthBase): - def __call__(self, r): - r.headers['Dummy-Auth-Test'] = 'dummy-auth-test-ok' - return r - - req = requests.Request('GET', httpbin('headers')) - assert not req.auth - - s = requests.Session() - s.auth = DummyAuth() - - prep = s.prepare_request(req) - resp = s.send(prep) - - assert resp.json()['headers'][ - 'Dummy-Auth-Test'] == 'dummy-auth-test-ok' - - def test_prepare_request_with_bytestring_url(self): - req = requests.Request('GET', b'https://httpbin.org/') - s = requests.Session() - prep = s.prepare_request(req) - assert prep.url == "https://httpbin.org/" - - def test_links(self): - r = requests.Response() - r.headers = { - 'cache-control': 'public, max-age=60, s-maxage=60', - 'connection': 'keep-alive', - 'content-encoding': 'gzip', - 'content-type': 'application/json; charset=utf-8', - 'date': 'Sat, 26 Jan 2013 16:47:56 GMT', - 'etag': '"6ff6a73c0e446c1f61614769e3ceb778"', - 'last-modified': 'Sat, 26 Jan 2013 16:22:39 GMT', - 'link': ('; rel="next", ; ' - ' rel="last"'), - 'server': 'GitHub.com', - 'status': '200 OK', - 'vary': 'Accept', - 'x-content-type-options': 'nosniff', - 'x-github-media-type': 'github.beta', - 'x-ratelimit-limit': '60', - 'x-ratelimit-remaining': '57' - } - assert r.links['next']['rel'] == 'next' - - def test_cookie_parameters(self): - key = 'some_cookie' - value = 'some_value' - secure = True - domain = 'test.com' - rest = {'HttpOnly': True} - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value, secure=secure, domain=domain, rest=rest) - - assert len(jar) == 1 - assert 'some_cookie' in jar - - cookie = list(jar)[0] - assert cookie.secure == secure - assert cookie.domain == domain - assert cookie._rest['HttpOnly'] == rest['HttpOnly'] - - def test_cookie_as_dict_keeps_len(self): - key = 'some_cookie' - value = 'some_value' - - key1 = 'some_cookie1' - value1 = 'some_value1' - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value) - jar.set(key1, value1) - - d1 = dict(jar) - d2 = dict(jar.iteritems()) - d3 = dict(jar.items()) - - assert len(jar) == 2 - assert len(d1) == 2 - assert len(d2) == 2 - assert len(d3) == 2 - - def test_cookie_as_dict_keeps_items(self): - key = 'some_cookie' - value = 'some_value' - - key1 = 'some_cookie1' - value1 = 'some_value1' - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value) - jar.set(key1, value1) - - d1 = dict(jar) - d2 = dict(jar.iteritems()) - d3 = dict(jar.items()) - - assert d1['some_cookie'] == 'some_value' - assert d2['some_cookie'] == 'some_value' - assert d3['some_cookie1'] == 'some_value1' - - def test_cookie_as_dict_keys(self): - key = 'some_cookie' - value = 'some_value' - - key1 = 'some_cookie1' - value1 = 'some_value1' - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value) - jar.set(key1, value1) - - keys = jar.keys() - assert keys == list(keys) - # make sure one can use keys multiple times - assert list(keys) == list(keys) - - def test_cookie_as_dict_values(self): - key = 'some_cookie' - value = 'some_value' - - key1 = 'some_cookie1' - value1 = 'some_value1' - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value) - jar.set(key1, value1) - - values = jar.values() - assert values == list(values) - # make sure one can use values multiple times - assert list(values) == list(values) - - def test_cookie_as_dict_items(self): - key = 'some_cookie' - value = 'some_value' - - key1 = 'some_cookie1' - value1 = 'some_value1' - - jar = requests.cookies.RequestsCookieJar() - jar.set(key, value) - jar.set(key1, value1) - - items = jar.items() - assert items == list(items) - # make sure one can use items multiple times - assert list(items) == list(items) - - def test_time_elapsed_blank(self, httpbin): - r = requests.get(httpbin('get')) - td = r.elapsed - total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) - * 10**6) / 10**6) - assert total_seconds > 0.0 - - def test_response_is_iterable(self): - r = requests.Response() - io = StringIO.StringIO('abc') - read_ = io.read - - def read_mock(amt, decode_content=None): - return read_(amt) - setattr(io, 'read', read_mock) - r.raw = io - assert next(iter(r)) - io.close() - - def test_response_decode_unicode(self): - """ - When called with decode_unicode, Response.iter_content should always - return unicode. - """ - r = requests.Response() - r._content_consumed = True - r._content = b'the content' - r.encoding = 'ascii' - - chunks = r.iter_content(decode_unicode=True) - assert all(isinstance(chunk, str) for chunk in chunks) - - # also for streaming - r = requests.Response() - r.raw = io.BytesIO(b'the content') - r.encoding = 'ascii' - chunks = r.iter_content(decode_unicode=True) - assert all(isinstance(chunk, str) for chunk in chunks) - - def test_request_and_response_are_pickleable(self, httpbin): - r = requests.get(httpbin('get')) - - # verify we can pickle the original request - assert pickle.loads(pickle.dumps(r.request)) - - # verify we can pickle the response and that we have access to - # the original request. - pr = pickle.loads(pickle.dumps(r)) - assert r.request.url == pr.request.url - assert r.request.headers == pr.request.headers - - def test_get_auth_from_url(self): - url = 'http://user:pass@complex.url.com/path?query=yes' - assert ('user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_spaces(self): - url = 'http://user:pass%20pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_not_encoded_spaces(self): - url = 'http://user:pass pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_percent_chars(self): - url = 'http://user%25user:pass@complex.url.com/path?query=yes' - assert ('user%user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_hashes(self): - url = 'http://user:pass%23pass@complex.url.com/path?query=yes' - assert ('user', 'pass#pass') == requests.utils.get_auth_from_url(url) - - def test_cannot_send_unprepared_requests(self, httpbin): - r = requests.Request(url=httpbin()) - with pytest.raises(ValueError): - requests.Session().send(r) - - def test_http_error(self): - error = requests.exceptions.HTTPError() - assert not error.response - response = requests.Response() - error = requests.exceptions.HTTPError(response=response) - assert error.response == response - error = requests.exceptions.HTTPError('message', response=response) - assert str(error) == 'message' - assert error.response == response - - def test_session_pickling(self, httpbin): - r = requests.Request('GET', httpbin('get')) - s = requests.Session() - - s = pickle.loads(pickle.dumps(s)) - s.proxies = getproxies() - - r = s.send(r.prepare()) - assert r.status_code == 200 - - def test_fixes_1329(self, httpbin): - """ - Ensure that header updates are done case-insensitively. - """ - s = requests.Session() - s.headers.update({'ACCEPT': 'BOGUS'}) - s.headers.update({'accept': 'application/json'}) - r = s.get(httpbin('get')) - headers = r.request.headers - assert headers['accept'] == 'application/json' - assert headers['Accept'] == 'application/json' - assert headers['ACCEPT'] == 'application/json' - - def test_uppercase_scheme_redirect(self, httpbin): - parts = urlparse(httpbin('html')) - url = "HTTP://" + parts.netloc + parts.path - r = requests.get(httpbin('redirect-to'), params={'url': url}) - assert r.status_code == 200 - assert r.url.lower() == url.lower() - - def test_transport_adapter_ordering(self): - s = requests.Session() - order = ['https://', 'http://'] - assert order == list(s.adapters) - s.mount('http://git', HTTPAdapter()) - s.mount('http://github', HTTPAdapter()) - s.mount('http://github.com', HTTPAdapter()) - s.mount('http://github.com/about/', HTTPAdapter()) - order = [ - 'http://github.com/about/', - 'http://github.com', - 'http://github', - 'http://git', - 'https://', - 'http://', - ] - assert order == list(s.adapters) - s.mount('http://gittip', HTTPAdapter()) - s.mount('http://gittip.com', HTTPAdapter()) - s.mount('http://gittip.com/about/', HTTPAdapter()) - order = [ - 'http://github.com/about/', - 'http://gittip.com/about/', - 'http://github.com', - 'http://gittip.com', - 'http://github', - 'http://gittip', - 'http://git', - 'https://', - 'http://', - ] - assert order == list(s.adapters) - s2 = requests.Session() - s2.adapters = {'http://': HTTPAdapter()} - s2.mount('https://', HTTPAdapter()) - assert 'http://' in s2.adapters - assert 'https://' in s2.adapters - - def test_header_remove_is_case_insensitive(self, httpbin): - # From issue #1321 - s = requests.Session() - s.headers['foo'] = 'bar' - r = s.get(httpbin('get'), headers={'FOO': None}) - assert 'foo' not in r.request.headers - - def test_params_are_merged_case_sensitive(self, httpbin): - s = requests.Session() - s.params['foo'] = 'bar' - r = s.get(httpbin('get'), params={'FOO': 'bar'}) - assert r.json()['args'] == {'foo': 'bar', 'FOO': 'bar'} - - def test_long_authinfo_in_url(self): - url = 'http://{0}:{1}@{2}:9000/path?query#frag'.format( - 'E8A3BE87-9E3F-4620-8858-95478E385B5B', - 'EA770032-DA4D-4D84-8CE9-29C6D910BF1E', - 'exactly-------------sixty-----------three------------characters', - ) - r = requests.Request('GET', url).prepare() - assert r.url == url - - def test_header_keys_are_native(self, httpbin): - headers = {u('unicode'): 'blah', 'byte'.encode('ascii'): 'blah'} - r = requests.Request('GET', httpbin('get'), headers=headers) - p = r.prepare() - - # This is testing that they are builtin strings. A bit weird, but there - # we go. - assert 'unicode' in p.headers.keys() - assert 'byte' in p.headers.keys() - - def test_can_send_nonstring_objects_with_files(self, httpbin): - data = {'a': 0.0} - files = {'b': 'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - - assert 'multipart/form-data' in p.headers['Content-Type'] - - def test_can_send_bytes_bytearray_objects_with_files(self, httpbin): - # Test bytes: - data = {'a': 'this is a string'} - files = {'b': b'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] - # Test bytearrays: - files = {'b': bytearray(b'foo')} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] - - def test_can_send_file_object_with_non_string_filename(self, httpbin): - f = io.BytesIO() - f.name = 2 - r = requests.Request('POST', httpbin('post'), files={'f': f}) - p = r.prepare() - - assert 'multipart/form-data' in p.headers['Content-Type'] - - def test_autoset_header_values_are_native(self, httpbin): - data = 'this is a string' - length = '16' - req = requests.Request('POST', httpbin('post'), data=data) - p = req.prepare() - - assert p.headers['Content-Length'] == length - - def test_nonhttp_schemes_dont_check_URLs(self): - test_urls = ( - 'data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==', - 'file:///etc/passwd', - 'magnet:?xt=urn:btih:be08f00302bc2d1d3cfa3af02024fa647a271431', - ) - for test_url in test_urls: - req = requests.Request('GET', test_url) - preq = req.prepare() - assert test_url == preq.url - - def test_auth_is_stripped_on_redirect_off_host(self, httpbin): - r = requests.get( - httpbin('redirect-to'), - params={'url': 'http://www.google.co.uk'}, - auth=('user', 'pass'), - ) - assert r.history[0].request.headers['Authorization'] - assert not r.request.headers.get('Authorization', '') - - def test_auth_is_retained_for_redirect_on_host(self, httpbin): - r = requests.get(httpbin('redirect/1'), auth=('user', 'pass')) - h1 = r.history[0].request.headers['Authorization'] - h2 = r.request.headers['Authorization'] - - assert h1 == h2 - - def test_manual_redirect_with_partial_body_read(self, httpbin): - s = requests.Session() - r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True) - assert r1.is_redirect - rg = s.resolve_redirects(r1, r1.request, stream=True) - - # read only the first eight bytes of the response body, - # then follow the redirect - r1.iter_content(8) - r2 = next(rg) - assert r2.is_redirect - - # read all of the response via iter_content, - # then follow the redirect - for _ in r2.iter_content(): - pass - r3 = next(rg) - assert not r3.is_redirect - - def _patch_adapter_gzipped_redirect(self, session, url): - adapter = session.get_adapter(url=url) - org_build_response = adapter.build_response - self._patched_response = False - - def build_response(*args, **kwargs): - resp = org_build_response(*args, **kwargs) - if not self._patched_response: - resp.raw.headers['content-encoding'] = 'gzip' - self._patched_response = True - return resp - - adapter.build_response = build_response - - def test_redirect_with_wrong_gzipped_header(self, httpbin): - s = requests.Session() - url = httpbin('redirect/1') - self._patch_adapter_gzipped_redirect(s, url) - s.get(url) - - def test_basic_auth_str_is_always_native(self): - s = _basic_auth_str("test", "test") - assert isinstance(s, builtin_str) - assert s == "Basic dGVzdDp0ZXN0" - - def test_requests_history_is_saved(self, httpbin): - r = requests.get(httpbin('redirect/5')) - total = r.history[-1].history - i = 0 - for item in r.history: - assert item.history == total[0:i] - i = i + 1 - - def test_json_param_post_content_type_works(self, httpbin): - r = requests.post( - httpbin('post'), - json={'life': 42} - ) - assert r.status_code == 200 - assert 'application/json' in r.request.headers['Content-Type'] - assert {'life': 42} == r.json()['json'] - - def test_json_param_post_should_not_override_data_param(self, httpbin): - r = requests.Request(method='POST', url=httpbin('post'), - data={'stuff': 'elixr'}, - json={'music': 'flute'}) - prep = r.prepare() - assert 'stuff=elixr' == prep.body - - def test_response_iter_lines(self, httpbin): - r = requests.get(httpbin('stream/4'), stream=True) - assert r.status_code == 200 - - it = r.iter_lines() - next(it) - assert len(list(it)) == 3 - - def test_unconsumed_session_response_closes_connection(self, httpbin): - s = requests.session() - - with contextlib.closing(s.get(httpbin('stream/4'), stream=True)) as response: - pass - - assert response._content_consumed is False - assert response.raw.closed - - @pytest.mark.xfail - def test_response_iter_lines_reentrant(self, httpbin): - """Response.iter_lines() is not reentrant safe""" - r = requests.get(httpbin('stream/4'), stream=True) - assert r.status_code == 200 - - next(r.iter_lines()) - assert len(list(r.iter_lines())) == 3 - - -class TestContentEncodingDetection(unittest.TestCase): - - def test_none(self): - encodings = requests.utils.get_encodings_from_content('') - assert not len(encodings) - - def test_html_charset(self): - """HTML5 meta charset attribute""" - content = '' - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_html4_pragma(self): - """HTML4 pragma directive""" - content = '' - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_xhtml_pragma(self): - """XHTML 1.x served with text/html MIME type""" - content = '' - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_xml(self): - """XHTML 1.x served as XML""" - content = '' - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_precedence(self): - content = ''' - - - - '''.strip() - encodings = requests.utils.get_encodings_from_content(content) - assert encodings == ['HTML5', 'HTML4', 'XML'] - - -class TestCaseInsensitiveDict(unittest.TestCase): - - def test_mapping_init(self): - cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}) - assert len(cid) == 2 - assert 'foo' in cid - assert 'bar' in cid - - def test_iterable_init(self): - cid = CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]) - assert len(cid) == 2 - assert 'foo' in cid - assert 'bar' in cid - - def test_kwargs_init(self): - cid = CaseInsensitiveDict(FOO='foo', BAr='bar') - assert len(cid) == 2 - assert 'foo' in cid - assert 'bar' in cid - - def test_docstring_example(self): - cid = CaseInsensitiveDict() - cid['Accept'] = 'application/json' - assert cid['aCCEPT'] == 'application/json' - assert list(cid) == ['Accept'] - - def test_len(self): - cid = CaseInsensitiveDict({'a': 'a', 'b': 'b'}) - cid['A'] = 'a' - assert len(cid) == 2 - - def test_getitem(self): - cid = CaseInsensitiveDict({'Spam': 'blueval'}) - assert cid['spam'] == 'blueval' - assert cid['SPAM'] == 'blueval' - - def test_fixes_649(self): - """__setitem__ should behave case-insensitively.""" - cid = CaseInsensitiveDict() - cid['spam'] = 'oneval' - cid['Spam'] = 'twoval' - cid['sPAM'] = 'redval' - cid['SPAM'] = 'blueval' - assert cid['spam'] == 'blueval' - assert cid['SPAM'] == 'blueval' - assert list(cid.keys()) == ['SPAM'] - - def test_delitem(self): - cid = CaseInsensitiveDict() - cid['Spam'] = 'someval' - del cid['sPam'] - assert 'spam' not in cid - assert len(cid) == 0 - - def test_contains(self): - cid = CaseInsensitiveDict() - cid['Spam'] = 'someval' - assert 'Spam' in cid - assert 'spam' in cid - assert 'SPAM' in cid - assert 'sPam' in cid - assert 'notspam' not in cid - - def test_get(self): - cid = CaseInsensitiveDict() - cid['spam'] = 'oneval' - cid['SPAM'] = 'blueval' - assert cid.get('spam') == 'blueval' - assert cid.get('SPAM') == 'blueval' - assert cid.get('sPam') == 'blueval' - assert cid.get('notspam', 'default') == 'default' - - def test_update(self): - cid = CaseInsensitiveDict() - cid['spam'] = 'blueval' - cid.update({'sPam': 'notblueval'}) - assert cid['spam'] == 'notblueval' - cid = CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}) - cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) - assert len(cid) == 2 - assert cid['foo'] == 'anotherfoo' - assert cid['bar'] == 'anotherbar' - - def test_update_retains_unchanged(self): - cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'}) - cid.update({'foo': 'newfoo'}) - assert cid['bar'] == 'bar' - - def test_iter(self): - cid = CaseInsensitiveDict({'Spam': 'spam', 'Eggs': 'eggs'}) - keys = frozenset(['Spam', 'Eggs']) - assert frozenset(iter(cid)) == keys - - def test_equality(self): - cid = CaseInsensitiveDict({'SPAM': 'blueval', 'Eggs': 'redval'}) - othercid = CaseInsensitiveDict({'spam': 'blueval', 'eggs': 'redval'}) - assert cid == othercid - del othercid['spam'] - assert cid != othercid - assert cid == {'spam': 'blueval', 'eggs': 'redval'} - assert cid != object() - - def test_setdefault(self): - cid = CaseInsensitiveDict({'Spam': 'blueval'}) - assert cid.setdefault('spam', 'notblueval') == 'blueval' - assert cid.setdefault('notspam', 'notblueval') == 'notblueval' - - def test_lower_items(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items()) - lowerkeyset = frozenset(['accept', 'user-agent']) - assert keyset == lowerkeyset - - def test_preserve_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - keyset = frozenset(['Accept', 'user-Agent']) - assert frozenset(i[0] for i in cid.items()) == keyset - assert frozenset(cid.keys()) == keyset - assert frozenset(cid) == keyset - - def test_preserve_last_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - cid.update({'ACCEPT': 'application/json'}) - cid['USER-AGENT'] = 'requests' - keyset = frozenset(['ACCEPT', 'USER-AGENT']) - assert frozenset(i[0] for i in cid.items()) == keyset - assert frozenset(cid.keys()) == keyset - assert frozenset(cid) == keyset - - def test_copy(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) - cid_copy = cid.copy() - assert cid == cid_copy - cid['changed'] = True - assert cid != cid_copy - - -class UtilsTestCase(unittest.TestCase): - - def test_super_len_io_streams(self): - """ Ensures that we properly deal with different kinds of IO streams. """ - # uses StringIO or io.StringIO (see import above) - from io import BytesIO - from requests.utils import super_len - - assert super_len(StringIO.StringIO()) == 0 - assert super_len( - StringIO.StringIO('with so much drama in the LBC')) == 29 - - assert super_len(BytesIO()) == 0 - assert super_len( - BytesIO(b"it's kinda hard bein' snoop d-o-double-g")) == 40 - - try: - import cStringIO - except ImportError: - pass - else: - assert super_len( - cStringIO.StringIO('but some how, some way...')) == 25 - - def test_super_len_correctly_calculates_len_of_partially_read_file(self): - """Ensure that we handle partially consumed file like objects.""" - from requests.utils import super_len - s = StringIO.StringIO() - s.write('foobarbogus') - assert super_len(s) == 0 - - def test_get_environ_proxies_ip_ranges(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" - assert get_environ_proxies('http://192.168.0.1:5000/') == {} - assert get_environ_proxies('http://192.168.0.1/') == {} - assert get_environ_proxies('http://172.16.1.1/') == {} - assert get_environ_proxies('http://172.16.1.1:5000/') == {} - assert get_environ_proxies('http://192.168.1.1:5000/') != {} - assert get_environ_proxies('http://192.168.1.1/') != {} - - def test_get_environ_proxies(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" - assert get_environ_proxies( - 'http://localhost.localdomain:5000/v1.0/') == {} - assert get_environ_proxies('http://www.requests.com/') != {} - - def test_select_proxies(self): - """Make sure we can select per-host proxies correctly.""" - from requests.utils import select_proxy - proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} - assert select_proxy('hTTp://u:p@Some.Host/path', proxies) == 'http://some.host.proxy' - assert select_proxy('hTTp://u:p@Other.Host/path', proxies) == 'http://http.proxy' - assert select_proxy('hTTps://Other.Host', proxies) is None - - def test_guess_filename_when_int(self): - from requests.utils import guess_filename - assert None is guess_filename(1) - - def test_guess_filename_when_filename_is_an_int(self): - from requests.utils import guess_filename - fake = type('Fake', (object,), {'name': 1})() - assert None is guess_filename(fake) - - def test_guess_filename_with_file_like_obj(self): - from requests.utils import guess_filename - from requests import compat - fake = type('Fake', (object,), {'name': b'value'})() - guessed_name = guess_filename(fake) - assert b'value' == guessed_name - assert isinstance(guessed_name, compat.bytes) - - def test_guess_filename_with_unicode_name(self): - from requests.utils import guess_filename - from requests import compat - filename = b'value'.decode('utf-8') - fake = type('Fake', (object,), {'name': filename})() - guessed_name = guess_filename(fake) - assert filename == guessed_name - assert isinstance(guessed_name, compat.str) - - def test_is_ipv4_address(self): - from requests.utils import is_ipv4_address - assert is_ipv4_address('8.8.8.8') - assert not is_ipv4_address('8.8.8.8.8') - assert not is_ipv4_address('localhost.localdomain') - - def test_is_valid_cidr(self): - from requests.utils import is_valid_cidr - assert not is_valid_cidr('8.8.8.8') - assert is_valid_cidr('192.168.1.0/24') - - def test_dotted_netmask(self): - from requests.utils import dotted_netmask - assert dotted_netmask(8) == '255.0.0.0' - assert dotted_netmask(24) == '255.255.255.0' - assert dotted_netmask(25) == '255.255.255.128' - - def test_address_in_network(self): - from requests.utils import address_in_network - assert address_in_network('192.168.1.1', '192.168.1.0/24') - assert not address_in_network('172.16.0.1', '192.168.1.0/24') - - def test_get_auth_from_url(self): - """Ensures that username and password in well-encoded URI as per - RFC 3986 are correclty extracted.""" - from requests.utils import get_auth_from_url - from requests.compat import quote - percent_encoding_test_chars = "%!*'();:@&=+$,/?#[] " - url_address = "request.com/url.html#test" - url = "http://" + quote( - percent_encoding_test_chars, '') + ':' + quote( - percent_encoding_test_chars, '') + '@' + url_address - (username, password) = get_auth_from_url(url) - assert username == percent_encoding_test_chars - assert password == percent_encoding_test_chars - - def test_requote_uri_with_unquoted_percents(self): - """Ensure we handle unquoted percent signs in redirects. - - See: https://github.com/kennethreitz/requests/issues/2356 - """ - from requests.utils import requote_uri - bad_uri = 'http://example.com/fiz?buz=%ppicture' - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(bad_uri) - - def test_requote_uri_properly_requotes(self): - """Ensure requoting doesn't break expectations.""" - from requests.utils import requote_uri - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(quoted) - - -class TestMorselToCookieExpires(unittest.TestCase): - - """Tests for morsel_to_cookie when morsel contains expires.""" - - def test_expires_valid_str(self): - """Test case where we convert expires from string time.""" - - morsel = Morsel() - morsel['expires'] = 'Thu, 01-Jan-1970 00:00:01 GMT' - cookie = morsel_to_cookie(morsel) - assert cookie.expires == 1 - - def test_expires_invalid_int(self): - """Test case where an invalid type is passed for expires.""" - - morsel = Morsel() - morsel['expires'] = 100 - with pytest.raises(TypeError): - morsel_to_cookie(morsel) - - def test_expires_invalid_str(self): - """Test case where an invalid string is input.""" - - morsel = Morsel() - morsel['expires'] = 'woops' - with pytest.raises(ValueError): - morsel_to_cookie(morsel) - - def test_expires_none(self): - """Test case where expires is None.""" - - morsel = Morsel() - morsel['expires'] = None - cookie = morsel_to_cookie(morsel) - assert cookie.expires is None - - -class TestMorselToCookieMaxAge(unittest.TestCase): - - """Tests for morsel_to_cookie when morsel contains max-age.""" - - def test_max_age_valid_int(self): - """Test case where a valid max age in seconds is passed.""" - - morsel = Morsel() - morsel['max-age'] = 60 - cookie = morsel_to_cookie(morsel) - assert isinstance(cookie.expires, int) - - def test_max_age_invalid_str(self): - """Test case where a invalid max age is passed.""" - - morsel = Morsel() - morsel['max-age'] = 'woops' - with pytest.raises(TypeError): - morsel_to_cookie(morsel) - - -class TestTimeout: - def test_stream_timeout(self, httpbin): - try: - requests.get(httpbin('delay/10'), timeout=2.0) - except requests.exceptions.Timeout as e: - assert 'Read timed out' in e.args[0].args[0] - - def test_invalid_timeout(self, httpbin): - with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout=(3, 4, 5)) - assert '(connect, read)' in str(e) - - with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout="foo") - assert 'must be an int or float' in str(e) - - def test_none_timeout(self, httpbin): - """ Check that you can set None as a valid timeout value. - - To actually test this behavior, we'd want to check that setting the - timeout to None actually lets the request block past the system default - timeout. However, this would make the test suite unbearably slow. - Instead we verify that setting the timeout to None does not prevent the - request from succeeding. - """ - r = requests.get(httpbin('get'), timeout=None) - assert r.status_code == 200 - - def test_read_timeout(self, httpbin): - try: - requests.get(httpbin('delay/10'), timeout=(None, 0.1)) - assert False, "The recv() request should time out." - except ReadTimeout: - pass - - def test_connect_timeout(self): - try: - requests.get(TARPIT, timeout=(0.1, None)) - assert False, "The connect() request should time out." - except ConnectTimeout as e: - assert isinstance(e, ConnectionError) - assert isinstance(e, Timeout) - - def test_total_timeout_connect(self): - try: - requests.get(TARPIT, timeout=(0.1, 0.1)) - assert False, "The connect() request should time out." - except ConnectTimeout: - pass - - def test_encoded_methods(self, httpbin): - """See: https://github.com/kennethreitz/requests/issues/2316""" - r = requests.request(b'GET', httpbin('get')) - assert r.ok - - -SendCall = collections.namedtuple('SendCall', ('args', 'kwargs')) - - -class RedirectSession(SessionRedirectMixin): - def __init__(self, order_of_redirects): - self.redirects = order_of_redirects - self.calls = [] - self.max_redirects = 30 - self.cookies = {} - self.trust_env = False - - def send(self, *args, **kwargs): - self.calls.append(SendCall(args, kwargs)) - return self.build_response() - - def build_response(self): - request = self.calls[-1].args[0] - r = requests.Response() - - try: - r.status_code = int(self.redirects.pop(0)) - except IndexError: - r.status_code = 200 - - r.headers = CaseInsensitiveDict({'Location': '/'}) - r.raw = self._build_raw() - r.request = request - return r - - def _build_raw(self): - string = StringIO.StringIO('') - setattr(string, 'release_conn', lambda *args: args) - return string - - -class TestRedirects: - default_keyword_args = { - 'stream': False, - 'verify': True, - 'cert': None, - 'timeout': None, - 'allow_redirects': False, - 'proxies': {}, - } - - def test_requests_are_updated_each_time(self, httpbin): - session = RedirectSession([303, 307]) - prep = requests.Request('POST', httpbin('post')).prepare() - r0 = session.send(prep) - assert r0.request.method == 'POST' - assert session.calls[-1] == SendCall((r0.request,), {}) - redirect_generator = session.resolve_redirects(r0, prep) - for response in redirect_generator: - assert response.request.method == 'GET' - send_call = SendCall((response.request,), - TestRedirects.default_keyword_args) - assert session.calls[-1] == send_call - - - -@pytest.fixture -def list_of_tuples(): - return [ - (('a', 'b'), ('c', 'd')), - (('c', 'd'), ('a', 'b')), - (('a', 'b'), ('c', 'd'), ('e', 'f')), - ] - - -def test_data_argument_accepts_tuples(list_of_tuples): - """ - Ensure that the data argument will accept tuples of strings - and properly encode them. - """ - for data in list_of_tuples: - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data=data, - hooks=default_hooks() - ) - assert p.body == urlencode(data) - - -def assert_copy(p, p_copy): - for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): - assert getattr(p, attr) == getattr(p_copy, attr) - - -def test_prepared_request_empty_copy(): - p = PreparedRequest() - assert_copy(p, p.copy()) - - -def test_prepared_request_no_cookies_copy(): - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data='foo=bar', - hooks=default_hooks() - ) - assert_copy(p, p.copy()) - - -def test_prepared_request_complete_copy(): - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data='foo=bar', - hooks=default_hooks(), - cookies={'foo': 'bar'} - ) - assert_copy(p, p.copy()) - - -def test_prepare_unicode_url(): - p = PreparedRequest() - p.prepare( - method='GET', - url=u('http://www.example.com/üniçø∂é'), - ) - assert_copy(p, p.copy()) - - -def test_urllib3_retries(httpbin): - from requests.packages.urllib3.util import Retry - s = requests.Session() - s.mount('http://', HTTPAdapter(max_retries=Retry( - total=2, status_forcelist=[500] - ))) - - with pytest.raises(RetryError): - s.get(httpbin('status/500')) - - -def test_urllib3_pool_connection_closed(httpbin): - s = requests.Session() - s.mount('http://', HTTPAdapter(pool_connections=0, pool_maxsize=0)) - - try: - s.get(httpbin('status/200')) - except ConnectionError as e: - assert u"Pool is closed." in str(e) - - -def test_vendor_aliases(): - from requests.packages import urllib3 - from requests.packages import chardet - - with pytest.raises(ImportError): - from requests.packages import webbrowser - - -if __name__ == '__main__': - unittest.main()