diff -Nru python-tornado-2.1.0/.gitignore python-tornado-3.1.1/.gitignore --- python-tornado-2.1.0/.gitignore 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ -*.pyc -*.so -*~ -build -dist/ -MANIFEST -tornado.egg-info -_auto2to3* -.tox \ No newline at end of file diff -Nru python-tornado-2.1.0/MANIFEST.in python-tornado-3.1.1/MANIFEST.in --- python-tornado-2.1.0/MANIFEST.in 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/MANIFEST.in 2013-08-04 19:34:21.000000000 +0000 @@ -1,8 +1,15 @@ recursive-include demos *.py *.yaml *.html *.css *.js *.xml *.sql README -include tornado/epoll.c include tornado/ca-certificates.crt include tornado/test/README +include tornado/test/csv_translations/fr_FR.csv +include tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.mo +include tornado/test/gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po +include tornado/test/options_test.cfg +include tornado/test/static/robots.txt +include tornado/test/static/dir/index.html +include tornado/test/templates/utf8.html include tornado/test/test.crt include tornado/test/test.key -include tornado/test/static/robots.txt +include README.rst +include runtests.sh global-exclude _auto2to3* \ No newline at end of file diff -Nru python-tornado-2.1.0/PKG-INFO python-tornado-3.1.1/PKG-INFO --- python-tornado-2.1.0/PKG-INFO 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/PKG-INFO 2013-09-01 18:44:12.000000000 +0000 @@ -0,0 +1,135 @@ +Metadata-Version: 1.1 +Name: tornado +Version: 3.1.1 +Summary: Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed. +Home-page: http://www.tornadoweb.org/ +Author: Facebook +Author-email: python-tornado@googlegroups.com +License: http://www.apache.org/licenses/LICENSE-2.0 +Description: Tornado Web Server + ================== + + `Tornado `_ is a Python web framework and + asynchronous networking library, originally developed at `FriendFeed + `_. By using non-blocking network I/O, Tornado + can scale to tens of thousands of open connections, making it ideal for + `long polling `_, + `WebSockets `_, and other + applications that require a long-lived connection to each user. + + + Quick links + ----------- + + * `Documentation `_ + * `Source (github) `_ + * `Mailing list `_ + * `Wiki `_ + + Hello, world + ------------ + + Here is a simple "Hello, world" example web app for Tornado:: + + import tornado.ioloop + import tornado.web + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + application = tornado.web.Application([ + (r"/", MainHandler), + ]) + + if __name__ == "__main__": + application.listen(8888) + tornado.ioloop.IOLoop.instance().start() + + This example does not use any of Tornado's asynchronous features; for + that see this `simple chat room + `_. + + Installation + ------------ + + **Automatic installation**:: + + pip install tornado + + Tornado is listed in `PyPI `_ and + can be installed with ``pip`` or ``easy_install``. Note that the + source distribution includes demo applications that are not present + when Tornado is installed in this way, so you may wish to download a + copy of the source tarball as well. + + **Manual installation**: Download the latest source from `PyPI + `_. + + .. parsed-literal:: + + tar xvzf tornado-$VERSION.tar.gz + cd tornado-$VERSION + python setup.py build + sudo python setup.py install + + The Tornado source code is `hosted on GitHub + `_. + + **Prerequisites**: Tornado runs on Python 2.6, 2.7, 3.2, and 3.3. It has + no strict dependencies outside the Python standard library, although some + features may require one of the following libraries: + + * `unittest2 `_ is needed to run + Tornado's test suite on Python 2.6 (it is unnecessary on more recent + versions of Python) + * `concurrent.futures `_ is the + recommended thread pool for use with Tornado and enables the use of + ``tornado.netutil.ThreadedResolver``. It is needed only on Python 2; + Python 3 includes this package in the standard library. + * `pycurl `_ is used by the optional + ``tornado.curl_httpclient``. Libcurl version 7.18.2 or higher is required; + version 7.21.1 or higher is recommended. + * `Twisted `_ may be used with the classes in + `tornado.platform.twisted`. + * `pycares `_ is an alternative + non-blocking DNS resolver that can be used when threads are not + appropriate. + * `Monotime `_ adds support for + a monotonic clock, which improves reliability in environments + where clock adjustments are frequent. No longer needed in Python 3.3. + + **Platforms**: Tornado should run on any Unix-like platform, although + for the best performance and scalability only Linux (with ``epoll``) + and BSD (with ``kqueue``) are recommended (even though Mac OS X is + derived from BSD and supports kqueue, its networking performance is + generally poor so it is recommended only for development use). + + Discussion and support + ---------------------- + + You can discuss Tornado on `the Tornado developer mailing list + `_, and report bugs on + the `GitHub issue tracker + `_. Links to additional + resources can be found on the `Tornado wiki + `_. + + Tornado is one of `Facebook's open source technologies + `_. It is available under + the `Apache License, Version 2.0 + `_. + + This web site and all documentation is licensed under `Creative + Commons 3.0 `_. + +Platform: UNKNOWN +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy diff -Nru python-tornado-2.1.0/README python-tornado-3.1.1/README --- python-tornado-2.1.0/README 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,32 +0,0 @@ -Tornado -======= -Tornado is an open source version of the scalable, non-blocking web server -and and tools that power FriendFeed. Documentation and downloads are -available at http://www.tornadoweb.org/ - -Tornado is licensed under the Apache Licence, Version 2.0 -(http://www.apache.org/licenses/LICENSE-2.0.html). - -Installation -============ -To install: - - python setup.py build - sudo python setup.py install - -Tornado has been tested on Python 2.5 and 2.6. To use all of the features -of Tornado, you need to have PycURL and (for Python 2.5 only) simplejson -installed. - -On Mac OS X 10.6, you can install the packages with: - - sudo easy_install pycurl - -On Ubuntu Linux, you can install the packages with: - - # Python 2.6 - sudo apt-get install python-pycurl - - # Python 2.5 - sudo apt-get install python-dev python-pycurl python-simplejson - diff -Nru python-tornado-2.1.0/README.rst python-tornado-3.1.1/README.rst --- python-tornado-2.1.0/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/README.rst 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,116 @@ +Tornado Web Server +================== + +`Tornado `_ is a Python web framework and +asynchronous networking library, originally developed at `FriendFeed +`_. By using non-blocking network I/O, Tornado +can scale to tens of thousands of open connections, making it ideal for +`long polling `_, +`WebSockets `_, and other +applications that require a long-lived connection to each user. + + +Quick links +----------- + +* `Documentation `_ +* `Source (github) `_ +* `Mailing list `_ +* `Wiki `_ + +Hello, world +------------ + +Here is a simple "Hello, world" example web app for Tornado:: + + import tornado.ioloop + import tornado.web + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + application = tornado.web.Application([ + (r"/", MainHandler), + ]) + + if __name__ == "__main__": + application.listen(8888) + tornado.ioloop.IOLoop.instance().start() + +This example does not use any of Tornado's asynchronous features; for +that see this `simple chat room +`_. + +Installation +------------ + +**Automatic installation**:: + + pip install tornado + +Tornado is listed in `PyPI `_ and +can be installed with ``pip`` or ``easy_install``. Note that the +source distribution includes demo applications that are not present +when Tornado is installed in this way, so you may wish to download a +copy of the source tarball as well. + +**Manual installation**: Download the latest source from `PyPI +`_. + +.. parsed-literal:: + + tar xvzf tornado-$VERSION.tar.gz + cd tornado-$VERSION + python setup.py build + sudo python setup.py install + +The Tornado source code is `hosted on GitHub +`_. + +**Prerequisites**: Tornado runs on Python 2.6, 2.7, 3.2, and 3.3. It has +no strict dependencies outside the Python standard library, although some +features may require one of the following libraries: + +* `unittest2 `_ is needed to run + Tornado's test suite on Python 2.6 (it is unnecessary on more recent + versions of Python) +* `concurrent.futures `_ is the + recommended thread pool for use with Tornado and enables the use of + ``tornado.netutil.ThreadedResolver``. It is needed only on Python 2; + Python 3 includes this package in the standard library. +* `pycurl `_ is used by the optional + ``tornado.curl_httpclient``. Libcurl version 7.18.2 or higher is required; + version 7.21.1 or higher is recommended. +* `Twisted `_ may be used with the classes in + `tornado.platform.twisted`. +* `pycares `_ is an alternative + non-blocking DNS resolver that can be used when threads are not + appropriate. +* `Monotime `_ adds support for + a monotonic clock, which improves reliability in environments + where clock adjustments are frequent. No longer needed in Python 3.3. + +**Platforms**: Tornado should run on any Unix-like platform, although +for the best performance and scalability only Linux (with ``epoll``) +and BSD (with ``kqueue``) are recommended (even though Mac OS X is +derived from BSD and supports kqueue, its networking performance is +generally poor so it is recommended only for development use). + +Discussion and support +---------------------- + +You can discuss Tornado on `the Tornado developer mailing list +`_, and report bugs on +the `GitHub issue tracker +`_. Links to additional +resources can be found on the `Tornado wiki +`_. + +Tornado is one of `Facebook's open source technologies +`_. It is available under +the `Apache License, Version 2.0 +`_. + +This web site and all documentation is licensed under `Creative +Commons 3.0 `_. diff -Nru python-tornado-2.1.0/debian/changelog python-tornado-3.1.1/debian/changelog --- python-tornado-2.1.0/debian/changelog 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/changelog 2013-11-04 00:52:36.000000000 +0000 @@ -1,3 +1,98 @@ +python-tornado (3.1.1-1~ubuntu12.04~ppa1) precise; urgency=low + + * No-change backport to precise + + -- Julian Taylor Mon, 04 Nov 2013 01:41:19 +0100 + +python-tornado (3.1.1-1) unstable; urgency=low + + * New upstream release + - fixes localhost binding on disabled network (Closes: #711806) + * drop upstream applied random-port.patch and CVE-2013-2099.patch + * bump standard to 3.9.5 no changes required + + -- Julian Taylor Sun, 03 Nov 2013 18:48:51 +0100 + +python-tornado (2.4.1-3) unstable; urgency=low + + * Team upload. + + [ Jakub Wilk ] + * Run tests only if DEB_BUILD_OPTIONS=nocheck is not set. + + [ Andrew Starr-Bochicchio ] + * Backport fix for CVE 2013-2009. Avoid allowing multiple + wildcards in a single SSL cert hostname segment (Closes: #709069). + + -- Andrew Starr-Bochicchio Fri, 04 Oct 2013 19:18:05 -0400 + +python-tornado (2.4.1-2) unstable; urgency=low + + [ Julian Taylor ] + * upload to unstable + * debian/rules: replace PWD with CURDIR + + [ Jakub Wilk ] + * Use canonical URIs for Vcs-* fields. + + -- Julian Taylor Thu, 09 May 2013 20:08:11 +0200 + +python-tornado (2.4.1-1) experimental; urgency=low + + * New upstream release (Closes: #698566) + * run python2 test during build (Closes: #663928) + * add autopkgtests + * ignoreuserwarning.patch: + disable warning in test to allow testing with the package already installed + * bump standards to 3.9.4, no changes required + * random-port.patch: get a random free port for tests + * don't compress example python files + + -- Julian Taylor Wed, 23 Jan 2013 20:26:46 +0100 + +python-tornado (2.3-2) unstable; urgency=low + + * debian/control + - Replaced the python3.2 in the Depends field of python3-tornado + by $(python3:Depends) + * Renamed python-tornado.examples to examples + * Removed python3-tornado.examples + + -- Carl Chenet Sun, 10 Jun 2012 01:26:50 +0200 + +python-tornado (2.3-1) unstable; urgency=low + + [ Julian Taylor ] + * Team upload + * new upstream release + - change debian/watch to download section instead of tag tarballs + tag tarball contain lots of unnecessary website and test data + - drop upstream applied CVE-2012-2374.patch + - refresh patches + - closes LP: #998615 + [ Carl Chenet] + * debian/control + - Replaced chaica@ohmytux.com by chaica@debian.org + - Replace (python3:Depends) in python3-tornado Depends field + because of https://github.com/facebook/tornado/issues/450 + - Modified the short description of python3-tornado package + * Renamed examples to python-tornado.examples + * Added python3-tornado.examples + + -- Carl Chenet Sun, 10 Jun 2012 00:12:24 +0200 + +python-tornado (2.1.0-3) unstable; urgency=high + + [ Julian Taylor ] + * Team upload + * bump X-Python-Version to >= 2.6 + lower versions require arch any for custom epoll extension + * move to Section: web (Closes: #665854) + - override lintian wrong-section-according-to-package-name + * CVE-2012-2374.patch adopted from upstream's GIT (Closes: #673987) + + -- Yaroslav Halchenko Tue, 22 May 2012 19:11:51 -0400 + python-tornado (2.1.0-2) unstable; urgency=low [ Thomas Kluyver ] diff -Nru python-tornado-2.1.0/debian/control python-tornado-3.1.1/debian/control --- python-tornado-2.1.0/debian/control 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/control 2013-11-03 19:52:46.000000000 +0000 @@ -1,19 +1,21 @@ Source: python-tornado -Section: python -X-Python-Version: >= 2.5 +Section: web +X-Python-Version: >= 2.6 X-Python3-Version: >= 3.2 Priority: optional Maintainer: Debian Python Modules Team -Uploaders: Carl Chenet , - Yaroslav Halchenko +Uploaders: Carl Chenet , + Yaroslav Halchenko , + Julian Taylor Build-Depends: debhelper (>= 7.0.50~), python-all (>= 2.6.6-3), python3-all, python3-setuptools -Standards-Version: 3.9.3 +Standards-Version: 3.9.5 Homepage: http://www.tornadoweb.org/ -Vcs-Svn: svn://svn.debian.org/python-modules/packages/python-tornado/trunk/ -Vcs-Browser: http://svn.debian.org/viewsvn/python-modules/packages/python-tornado/trunk/ +Vcs-Svn: svn://anonscm.debian.org/python-modules/packages/python-tornado/trunk/ +Vcs-Browser: http://anonscm.debian.org/viewvc/python-modules/packages/python-tornado/trunk/ +XS-Testsuite: autopkgtest Package: python-tornado Architecture: all @@ -35,7 +37,7 @@ Package: python3-tornado Architecture: all Depends: ca-certificates, ${misc:Depends}, ${python3:Depends} -Description: scalable, non-blocking web server and tools +Description: scalable, non-blocking web server and tools - Python 3 package Tornado is an open source version of the scalable, non-blocking web server and tools that power FriendFeed. The FriendFeed application is written using a web framework that looks a bit like web.py or diff -Nru python-tornado-2.1.0/debian/patches/certs-path.patch python-tornado-3.1.1/debian/patches/certs-path.patch --- python-tornado-2.1.0/debian/patches/certs-path.patch 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/patches/certs-path.patch 2013-11-03 19:52:46.000000000 +0000 @@ -3,9 +3,9 @@ --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py -@@ -31,7 +31,10 @@ try: +@@ -30,7 +30,10 @@ try: except ImportError: - ssl = None + import urllib.parse as urlparse # py3 -_DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' +if os.path.exists('/etc/ssl/certs/ca-certificates.crt'): @@ -13,5 +13,5 @@ +else: + _DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' + class SimpleAsyncHTTPClient(AsyncHTTPClient): - """Non-blocking HTTP client with no external dependencies. diff -Nru python-tornado-2.1.0/debian/patches/ignore-ca-certificates.patch python-tornado-3.1.1/debian/patches/ignore-ca-certificates.patch --- python-tornado-2.1.0/debian/patches/ignore-ca-certificates.patch 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/patches/ignore-ca-certificates.patch 2013-11-03 19:52:46.000000000 +0000 @@ -3,12 +3,12 @@ --- a/setup.py +++ b/setup.py -@@ -44,7 +44,7 @@ distutils.core.setup( +@@ -35,7 +35,7 @@ distutils.core.setup( version=version, packages = ["tornado", "tornado.test", "tornado.platform"], package_data = { - "tornado": ["ca-certificates.crt"], -+ # "tornado": ["ca-certificates.crt"], - "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"], - }, - ext_modules = extensions, ++ #"tornado": ["ca-certificates.crt"], + # data files need to be listed both here (which determines what gets + # installed) and in MANIFEST.in (which determines what gets included + # in the sdist tarball) diff -Nru python-tornado-2.1.0/debian/patches/ignoreuserwarning.patch python-tornado-3.1.1/debian/patches/ignoreuserwarning.patch --- python-tornado-2.1.0/debian/patches/ignoreuserwarning.patch 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/debian/patches/ignoreuserwarning.patch 2013-11-03 19:52:46.000000000 +0000 @@ -0,0 +1,13 @@ +Description: ignore userwarning in tests + Required to run tests from source with the package already installed. + Else one gets check_version_conflict warning from pkg_resources. +--- a/tornado/test/runtests.py ++++ b/tornado/test/runtests.py +@@ -72,6 +72,7 @@ if __name__ == '__main__': + # setuptools sometimes gives ImportWarnings about things that are on + # sys.path even if they're not being used. + warnings.filterwarnings("ignore", category=ImportWarning) ++ warnings.filterwarnings("ignore", category=UserWarning) + # Tornado generally shouldn't use anything deprecated, but some of + # our dependencies do (last match wins). + warnings.filterwarnings("ignore", category=DeprecationWarning) diff -Nru python-tornado-2.1.0/debian/patches/series python-tornado-3.1.1/debian/patches/series --- python-tornado-2.1.0/debian/patches/series 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/patches/series 2013-11-03 19:52:46.000000000 +0000 @@ -1,2 +1,3 @@ ignore-ca-certificates.patch certs-path.patch +ignoreuserwarning.patch diff -Nru python-tornado-2.1.0/debian/python-tornado.lintian-overrides python-tornado-3.1.1/debian/python-tornado.lintian-overrides --- python-tornado-2.1.0/debian/python-tornado.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/debian/python-tornado.lintian-overrides 2013-11-03 19:52:46.000000000 +0000 @@ -0,0 +1,2 @@ +# webserver belongs to web (#665854) +python-tornado binary: wrong-section-according-to-package-name diff -Nru python-tornado-2.1.0/debian/rules python-tornado-3.1.1/debian/rules --- python-tornado-2.1.0/debian/rules 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/rules 2013-11-03 19:52:46.000000000 +0000 @@ -25,3 +25,14 @@ dh_auto_clean rm -rf build rm -rf *.egg-info + +override_dh_auto_test: +ifeq "$(filter nocheck,$(DEB_BUILD_OPTIONS))" "" + # python3 tests only in autopkgtest because I'm lazy + set -e && for pyvers in $(PYTHON2); do \ + PYTHONPATH=$(CURDIR) python$$pyvers ./tornado/test/runtests.py; \ + done +endif + +override_dh_compress: + dh_compress -X.py diff -Nru python-tornado-2.1.0/debian/tests/control python-tornado-3.1.1/debian/tests/control --- python-tornado-2.1.0/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/debian/tests/control 2013-11-03 19:52:46.000000000 +0000 @@ -0,0 +1,6 @@ +Tests: python2 +Depends: python-tornado, python-all, python-twisted, python-pycurl + +#py3curl broken (https://github.com/facebook/tornado/issues/671) +Tests: python3 +Depends: python3-tornado, python3-all diff -Nru python-tornado-2.1.0/debian/tests/python2 python-tornado-3.1.1/debian/tests/python2 --- python-tornado-2.1.0/debian/tests/python2 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/debian/tests/python2 2013-11-03 19:52:46.000000000 +0000 @@ -0,0 +1,11 @@ +#!/bin/sh +set -efu + +PYS=${PYS:-"$(pyversions -r 2>/dev/null)"} + +cd "$ADTTMP" + +for py in $PYS; do + echo "=== $py ===" + $py /usr/lib/$py/dist-packages/tornado/test/runtests.py 2>&1 +done diff -Nru python-tornado-2.1.0/debian/tests/python3 python-tornado-3.1.1/debian/tests/python3 --- python-tornado-2.1.0/debian/tests/python3 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/debian/tests/python3 2013-11-03 19:52:46.000000000 +0000 @@ -0,0 +1,11 @@ +#!/bin/sh +set -efu + +PYS=${PYS:-"$(py3versions -r 2>/dev/null)"} + +cd "$ADTTMP" + +for py in $PYS; do + echo "=== $py ===" + $py /usr/lib/python3/dist-packages/tornado/test/runtests.py 2>&1 +done diff -Nru python-tornado-2.1.0/debian/watch python-tornado-3.1.1/debian/watch --- python-tornado-2.1.0/debian/watch 2012-03-13 22:12:09.000000000 +0000 +++ python-tornado-3.1.1/debian/watch 2013-11-03 19:52:46.000000000 +0000 @@ -1,2 +1,2 @@ version=3 -http://githubredir.debian.net/github/facebook/tornado /v(.*).tar.gz +https://github.com/facebook/tornado/tags .*/v(.*).tar.gz diff -Nru python-tornado-2.1.0/demos/appengine/app.yaml python-tornado-3.1.1/demos/appengine/app.yaml --- python-tornado-2.1.0/demos/appengine/app.yaml 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/app.yaml 2013-08-04 19:34:21.000000000 +0000 @@ -1,11 +1,12 @@ application: tornado-appengine -version: 1 -runtime: python +version: 2 +runtime: python27 api_version: 1 +threadsafe: yes handlers: - url: /static/ static_dir: static - url: /.* - script: blog.py + script: blog.application diff -Nru python-tornado-2.1.0/demos/appengine/blog.py python-tornado-3.1.1/demos/appengine/blog.py --- python-tornado-2.1.0/demos/appengine/blog.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/blog.py 2013-08-04 19:34:21.000000000 +0000 @@ -15,13 +15,12 @@ # under the License. import functools -import markdown import os.path import re +import tornado.escape import tornado.web import tornado.wsgi import unicodedata -import wsgiref.handlers from google.appengine.api import users from google.appengine.ext import db @@ -32,7 +31,7 @@ author = db.UserProperty() title = db.StringProperty(required=True) slug = db.StringProperty(required=True) - markdown = db.TextProperty(required=True) + body_source = db.TextProperty(required=True) html = db.TextProperty(required=True) published = db.DateTimeProperty(auto_now_add=True) updated = db.DateTimeProperty(auto_now=True) @@ -67,10 +66,11 @@ def get_login_url(self): return users.create_login_url(self.request.uri) - def render_string(self, template_name, **kwargs): + def get_template_namespace(self): # Let the templates access the users module to generate login URLs - return tornado.web.RequestHandler.render_string( - self, template_name, users=users, **kwargs) + ns = super(BaseHandler, self).get_template_namespace() + ns['users'] = users + return ns class HomeHandler(BaseHandler): @@ -116,8 +116,9 @@ if key: entry = Entry.get(key) entry.title = self.get_argument("title") - entry.markdown = self.get_argument("markdown") - entry.html = markdown.markdown(self.get_argument("markdown")) + entry.body_source = self.get_argument("body_source") + entry.html = tornado.escape.linkify( + self.get_argument("body_source")) else: title = self.get_argument("title") slug = unicodedata.normalize("NFKD", title).encode( @@ -134,8 +135,8 @@ author=self.current_user, title=title, slug=slug, - markdown=self.get_argument("markdown"), - html=markdown.markdown(self.get_argument("markdown")), + body_source=self.get_argument("body_source"), + html=tornado.escape.linkify(self.get_argument("body_source")), ) entry.put() self.redirect("/entry/" + entry.slug) @@ -151,7 +152,6 @@ "template_path": os.path.join(os.path.dirname(__file__), "templates"), "ui_modules": {"Entry": EntryModule}, "xsrf_cookies": True, - "autoescape": None, } application = tornado.wsgi.WSGIApplication([ (r"/", HomeHandler), @@ -160,11 +160,3 @@ (r"/entry/([^/]+)", EntryHandler), (r"/compose", ComposeHandler), ], **settings) - - -def main(): - wsgiref.handlers.CGIHandler().run(application) - - -if __name__ == "__main__": - main() diff -Nru python-tornado-2.1.0/demos/appengine/markdown.py python-tornado-3.1.1/demos/appengine/markdown.py --- python-tornado-2.1.0/demos/appengine/markdown.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/markdown.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,1877 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2007-2008 ActiveState Corp. -# License: MIT (http://www.opensource.org/licenses/mit-license.php) - -r"""A fast and complete Python implementation of Markdown. - -[from http://daringfireball.net/projects/markdown/] -> Markdown is a text-to-HTML filter; it translates an easy-to-read / -> easy-to-write structured text format into HTML. Markdown's text -> format is most similar to that of plain text email, and supports -> features such as headers, *emphasis*, code blocks, blockquotes, and -> links. -> -> Markdown's syntax is designed not as a generic markup language, but -> specifically to serve as a front-end to (X)HTML. You can use span-level -> HTML tags anywhere in a Markdown document, and you can use block level -> HTML tags (like
and as well). - -Module usage: - - >>> import markdown2 - >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)` - u'

boo!

\n' - - >>> markdowner = Markdown() - >>> markdowner.convert("*boo!*") - u'

boo!

\n' - >>> markdowner.convert("**boom!**") - u'

boom!

\n' - -This implementation of Markdown implements the full "core" syntax plus a -number of extras (e.g., code syntax coloring, footnotes) as described on -. -""" - -cmdln_desc = """A fast and complete Python implementation of Markdown, a -text-to-HTML conversion tool for web writers. -""" - -# Dev Notes: -# - There is already a Python markdown processor -# (http://www.freewisdom.org/projects/python-markdown/). -# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm -# not yet sure if there implications with this. Compare 'pydoc sre' -# and 'perldoc perlre'. - -__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl -__version__ = '1.0.1.14' -__author__ = "Trent Mick" - -import os -import sys -from pprint import pprint -import re -import logging -try: - from hashlib import md5 -except ImportError: - from md5 import md5 -import optparse -from random import random -import codecs - - - -#---- Python version compat - -if sys.version_info[:2] < (2,4): - from sets import Set as set - def reversed(sequence): - for i in sequence[::-1]: - yield i - def _unicode_decode(s, encoding, errors='xmlcharrefreplace'): - return unicode(s, encoding, errors) -else: - def _unicode_decode(s, encoding, errors='strict'): - return s.decode(encoding, errors) - - -#---- globals - -DEBUG = False -log = logging.getLogger("markdown") - -DEFAULT_TAB_WIDTH = 4 - -# Table of hash values for escaped characters: -def _escape_hash(s): - # Lame attempt to avoid possible collision with someone actually - # using the MD5 hexdigest of one of these chars in there text. - # Other ideas: random.random(), uuid.uuid() - #return md5(s).hexdigest() # Markdown.pl effectively does this. - return 'md5-'+md5(s).hexdigest() -g_escape_table = dict([(ch, _escape_hash(ch)) - for ch in '\\`*_{}[]()>#+-.!']) - - - -#---- exceptions - -class MarkdownError(Exception): - pass - - - -#---- public api - -def markdown_path(path, encoding="utf-8", - html4tags=False, tab_width=DEFAULT_TAB_WIDTH, - safe_mode=None, extras=None, link_patterns=None, - use_file_vars=False): - text = codecs.open(path, 'r', encoding).read() - return Markdown(html4tags=html4tags, tab_width=tab_width, - safe_mode=safe_mode, extras=extras, - link_patterns=link_patterns, - use_file_vars=use_file_vars).convert(text) - -def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH, - safe_mode=None, extras=None, link_patterns=None, - use_file_vars=False): - return Markdown(html4tags=html4tags, tab_width=tab_width, - safe_mode=safe_mode, extras=extras, - link_patterns=link_patterns, - use_file_vars=use_file_vars).convert(text) - -class Markdown(object): - # The dict of "extras" to enable in processing -- a mapping of - # extra name to argument for the extra. Most extras do not have an - # argument, in which case the value is None. - # - # This can be set via (a) subclassing and (b) the constructor - # "extras" argument. - extras = None - - urls = None - titles = None - html_blocks = None - html_spans = None - html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py - - # Used to track when we're inside an ordered or unordered list - # (see _ProcessListItems() for details): - list_level = 0 - - _ws_only_line_re = re.compile(r"^[ \t]+$", re.M) - - def __init__(self, html4tags=False, tab_width=4, safe_mode=None, - extras=None, link_patterns=None, use_file_vars=False): - if html4tags: - self.empty_element_suffix = ">" - else: - self.empty_element_suffix = " />" - self.tab_width = tab_width - - # For compatibility with earlier markdown2.py and with - # markdown.py's safe_mode being a boolean, - # safe_mode == True -> "replace" - if safe_mode is True: - self.safe_mode = "replace" - else: - self.safe_mode = safe_mode - - if self.extras is None: - self.extras = {} - elif not isinstance(self.extras, dict): - self.extras = dict([(e, None) for e in self.extras]) - if extras: - if not isinstance(extras, dict): - extras = dict([(e, None) for e in extras]) - self.extras.update(extras) - assert isinstance(self.extras, dict) - self._instance_extras = self.extras.copy() - self.link_patterns = link_patterns - self.use_file_vars = use_file_vars - self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M) - - def reset(self): - self.urls = {} - self.titles = {} - self.html_blocks = {} - self.html_spans = {} - self.list_level = 0 - self.extras = self._instance_extras.copy() - if "footnotes" in self.extras: - self.footnotes = {} - self.footnote_ids = [] - - def convert(self, text): - """Convert the given text.""" - # Main function. The order in which other subs are called here is - # essential. Link and image substitutions need to happen before - # _EscapeSpecialChars(), so that any *'s or _'s in the - # and tags get encoded. - - # Clear the global hashes. If we don't clear these, you get conflicts - # from other articles when generating a page which contains more than - # one article (e.g. an index page that shows the N most recent - # articles): - self.reset() - - if not isinstance(text, unicode): - #TODO: perhaps shouldn't presume UTF-8 for string input? - text = unicode(text, 'utf-8') - - if self.use_file_vars: - # Look for emacs-style file variable hints. - emacs_vars = self._get_emacs_vars(text) - if "markdown-extras" in emacs_vars: - splitter = re.compile("[ ,]+") - for e in splitter.split(emacs_vars["markdown-extras"]): - if '=' in e: - ename, earg = e.split('=', 1) - try: - earg = int(earg) - except ValueError: - pass - else: - ename, earg = e, None - self.extras[ename] = earg - - # Standardize line endings: - text = re.sub("\r\n|\r", "\n", text) - - # Make sure $text ends with a couple of newlines: - text += "\n\n" - - # Convert all tabs to spaces. - text = self._detab(text) - - # Strip any lines consisting only of spaces and tabs. - # This makes subsequent regexen easier to write, because we can - # match consecutive blank lines with /\n+/ instead of something - # contorted like /[ \t]*\n+/ . - text = self._ws_only_line_re.sub("", text) - - if self.safe_mode: - text = self._hash_html_spans(text) - - # Turn block-level HTML blocks into hash entries - text = self._hash_html_blocks(text, raw=True) - - # Strip link definitions, store in hashes. - if "footnotes" in self.extras: - # Must do footnotes first because an unlucky footnote defn - # looks like a link defn: - # [^4]: this "looks like a link defn" - text = self._strip_footnote_definitions(text) - text = self._strip_link_definitions(text) - - text = self._run_block_gamut(text) - - if "footnotes" in self.extras: - text = self._add_footnotes(text) - - text = self._unescape_special_chars(text) - - if self.safe_mode: - text = self._unhash_html_spans(text) - - text += "\n" - return text - - _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE) - # This regular expression is intended to match blocks like this: - # PREFIX Local Variables: SUFFIX - # PREFIX mode: Tcl SUFFIX - # PREFIX End: SUFFIX - # Some notes: - # - "[ \t]" is used instead of "\s" to specifically exclude newlines - # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does - # not like anything other than Unix-style line terminators. - _emacs_local_vars_pat = re.compile(r"""^ - (?P(?:[^\r\n|\n|\r])*?) - [\ \t]*Local\ Variables:[\ \t]* - (?P.*?)(?:\r\n|\n|\r) - (?P.*?\1End:) - """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE) - - def _get_emacs_vars(self, text): - """Return a dictionary of emacs-style local variables. - - Parsing is done loosely according to this spec (and according to - some in-practice deviations from this): - http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables - """ - emacs_vars = {} - SIZE = pow(2, 13) # 8kB - - # Search near the start for a '-*-'-style one-liner of variables. - head = text[:SIZE] - if "-*-" in head: - match = self._emacs_oneliner_vars_pat.search(head) - if match: - emacs_vars_str = match.group(1) - assert '\n' not in emacs_vars_str - emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';') - if s.strip()] - if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]: - # While not in the spec, this form is allowed by emacs: - # -*- Tcl -*- - # where the implied "variable" is "mode". This form - # is only allowed if there are no other variables. - emacs_vars["mode"] = emacs_var_strs[0].strip() - else: - for emacs_var_str in emacs_var_strs: - try: - variable, value = emacs_var_str.strip().split(':', 1) - except ValueError: - log.debug("emacs variables error: malformed -*- " - "line: %r", emacs_var_str) - continue - # Lowercase the variable name because Emacs allows "Mode" - # or "mode" or "MoDe", etc. - emacs_vars[variable.lower()] = value.strip() - - tail = text[-SIZE:] - if "Local Variables" in tail: - match = self._emacs_local_vars_pat.search(tail) - if match: - prefix = match.group("prefix") - suffix = match.group("suffix") - lines = match.group("content").splitlines(0) - #print "prefix=%r, suffix=%r, content=%r, lines: %s"\ - # % (prefix, suffix, match.group("content"), lines) - - # Validate the Local Variables block: proper prefix and suffix - # usage. - for i, line in enumerate(lines): - if not line.startswith(prefix): - log.debug("emacs variables error: line '%s' " - "does not use proper prefix '%s'" - % (line, prefix)) - return {} - # Don't validate suffix on last line. Emacs doesn't care, - # neither should we. - if i != len(lines)-1 and not line.endswith(suffix): - log.debug("emacs variables error: line '%s' " - "does not use proper suffix '%s'" - % (line, suffix)) - return {} - - # Parse out one emacs var per line. - continued_for = None - for line in lines[:-1]: # no var on the last line ("PREFIX End:") - if prefix: line = line[len(prefix):] # strip prefix - if suffix: line = line[:-len(suffix)] # strip suffix - line = line.strip() - if continued_for: - variable = continued_for - if line.endswith('\\'): - line = line[:-1].rstrip() - else: - continued_for = None - emacs_vars[variable] += ' ' + line - else: - try: - variable, value = line.split(':', 1) - except ValueError: - log.debug("local variables error: missing colon " - "in local variables entry: '%s'" % line) - continue - # Do NOT lowercase the variable name, because Emacs only - # allows "mode" (and not "Mode", "MoDe", etc.) in this block. - value = value.strip() - if value.endswith('\\'): - value = value[:-1].rstrip() - continued_for = variable - else: - continued_for = None - emacs_vars[variable] = value - - # Unquote values. - for var, val in emacs_vars.items(): - if len(val) > 1 and (val.startswith('"') and val.endswith('"') - or val.startswith('"') and val.endswith('"')): - emacs_vars[var] = val[1:-1] - - return emacs_vars - - # Cribbed from a post by Bart Lateur: - # - _detab_re = re.compile(r'(.*?)\t', re.M) - def _detab_sub(self, match): - g1 = match.group(1) - return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width)) - def _detab(self, text): - r"""Remove (leading?) tabs from a file. - - >>> m = Markdown() - >>> m._detab("\tfoo") - ' foo' - >>> m._detab(" \tfoo") - ' foo' - >>> m._detab("\t foo") - ' foo' - >>> m._detab(" foo") - ' foo' - >>> m._detab(" foo\n\tbar\tblam") - ' foo\n bar blam' - """ - if '\t' not in text: - return text - return self._detab_re.subn(self._detab_sub, text)[0] - - _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del' - _strict_tag_block_re = re.compile(r""" - ( # save in \1 - ^ # start of line (with re.M) - <(%s) # start tag = \2 - \b # word break - (.*\n)*? # any number of lines, minimally matching - # the matching end tag - [ \t]* # trailing spaces/tabs - (?=\n+|\Z) # followed by a newline or end of document - ) - """ % _block_tags_a, - re.X | re.M) - - _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math' - _liberal_tag_block_re = re.compile(r""" - ( # save in \1 - ^ # start of line (with re.M) - <(%s) # start tag = \2 - \b # word break - (.*\n)*? # any number of lines, minimally matching - .* # the matching end tag - [ \t]* # trailing spaces/tabs - (?=\n+|\Z) # followed by a newline or end of document - ) - """ % _block_tags_b, - re.X | re.M) - - def _hash_html_block_sub(self, match, raw=False): - html = match.group(1) - if raw and self.safe_mode: - html = self._sanitize_html(html) - key = _hash_text(html) - self.html_blocks[key] = html - return "\n\n" + key + "\n\n" - - def _hash_html_blocks(self, text, raw=False): - """Hashify HTML blocks - - We only want to do this for block-level HTML tags, such as headers, - lists, and tables. That's because we still want to wrap

s around - "paragraphs" that are wrapped in non-block-level tags, such as anchors, - phrase emphasis, and spans. The list of tags we're looking for is - hard-coded. - - @param raw {boolean} indicates if these are raw HTML blocks in - the original source. It makes a difference in "safe" mode. - """ - if '<' not in text: - return text - - # Pass `raw` value into our calls to self._hash_html_block_sub. - hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw) - - # First, look for nested blocks, e.g.: - #

- #
- # tags for inner block must be indented. - #
- #
- # - # The outermost tags must start at the left margin for this to match, and - # the inner nested divs must be indented. - # We need to do this before the next, more liberal match, because the next - # match will start at the first `
` and stop at the first `
`. - text = self._strict_tag_block_re.sub(hash_html_block_sub, text) - - # Now match more liberally, simply from `\n` to `\n` - text = self._liberal_tag_block_re.sub(hash_html_block_sub, text) - - # Special case just for
. It was easier to make a special - # case than to make the other regex more complicated. - if "", start_idx) + 3 - except ValueError, ex: - break - - # Start position for next comment block search. - start = end_idx - - # Validate whitespace before comment. - if start_idx: - # - Up to `tab_width - 1` spaces before start_idx. - for i in range(self.tab_width - 1): - if text[start_idx - 1] != ' ': - break - start_idx -= 1 - if start_idx == 0: - break - # - Must be preceded by 2 newlines or hit the start of - # the document. - if start_idx == 0: - pass - elif start_idx == 1 and text[0] == '\n': - start_idx = 0 # to match minute detail of Markdown.pl regex - elif text[start_idx-2:start_idx] == '\n\n': - pass - else: - break - - # Validate whitespace after comment. - # - Any number of spaces and tabs. - while end_idx < len(text): - if text[end_idx] not in ' \t': - break - end_idx += 1 - # - Must be following by 2 newlines or hit end of text. - if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'): - continue - - # Escape and hash (must match `_hash_html_block_sub`). - html = text[start_idx:end_idx] - if raw and self.safe_mode: - html = self._sanitize_html(html) - key = _hash_text(html) - self.html_blocks[key] = html - text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:] - - if "xml" in self.extras: - # Treat XML processing instructions and namespaced one-liner - # tags as if they were block HTML tags. E.g., if standalone - # (i.e. are their own paragraph), the following do not get - # wrapped in a

tag: - # - # - # - _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width) - text = _xml_oneliner_re.sub(hash_html_block_sub, text) - - return text - - def _strip_link_definitions(self, text): - # Strips link definitions from text, stores the URLs and titles in - # hash references. - less_than_tab = self.tab_width - 1 - - # Link defs are in the form: - # [id]: url "optional title" - _link_def_re = re.compile(r""" - ^[ ]{0,%d}\[(.+)\]: # id = \1 - [ \t]* - \n? # maybe *one* newline - [ \t]* - ? # url = \2 - [ \t]* - (?: - \n? # maybe one newline - [ \t]* - (?<=\s) # lookbehind for whitespace - ['"(] - ([^\n]*) # title = \3 - ['")] - [ \t]* - )? # title is optional - (?:\n+|\Z) - """ % less_than_tab, re.X | re.M | re.U) - return _link_def_re.sub(self._extract_link_def_sub, text) - - def _extract_link_def_sub(self, match): - id, url, title = match.groups() - key = id.lower() # Link IDs are case-insensitive - self.urls[key] = self._encode_amps_and_angles(url) - if title: - self.titles[key] = title.replace('"', '"') - return "" - - def _extract_footnote_def_sub(self, match): - id, text = match.groups() - text = _dedent(text, skip_first_line=not text.startswith('\n')).strip() - normed_id = re.sub(r'\W', '-', id) - # Ensure footnote text ends with a couple newlines (for some - # block gamut matches). - self.footnotes[normed_id] = text + "\n\n" - return "" - - def _strip_footnote_definitions(self, text): - """A footnote definition looks like this: - - [^note-id]: Text of the note. - - May include one or more indented paragraphs. - - Where, - - The 'note-id' can be pretty much anything, though typically it - is the number of the footnote. - - The first paragraph may start on the next line, like so: - - [^note-id]: - Text of the note. - """ - less_than_tab = self.tab_width - 1 - footnote_def_re = re.compile(r''' - ^[ ]{0,%d}\[\^(.+)\]: # id = \1 - [ \t]* - ( # footnote text = \2 - # First line need not start with the spaces. - (?:\s*.*\n+) - (?: - (?:[ ]{%d} | \t) # Subsequent lines must be indented. - .*\n+ - )* - ) - # Lookahead for non-space at line-start, or end of doc. - (?:(?=^[ ]{0,%d}\S)|\Z) - ''' % (less_than_tab, self.tab_width, self.tab_width), - re.X | re.M) - return footnote_def_re.sub(self._extract_footnote_def_sub, text) - - - _hr_res = [ - re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M), - re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M), - re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M), - ] - - def _run_block_gamut(self, text): - # These are all the transformations that form block-level - # tags like paragraphs, headers, and list items. - - text = self._do_headers(text) - - # Do Horizontal Rules: - hr = "\n tags around block-level tags. - text = self._hash_html_blocks(text) - - text = self._form_paragraphs(text) - - return text - - def _pyshell_block_sub(self, match): - lines = match.group(0).splitlines(0) - _dedentlines(lines) - indent = ' ' * self.tab_width - s = ('\n' # separate from possible cuddled paragraph - + indent + ('\n'+indent).join(lines) - + '\n\n') - return s - - def _prepare_pyshell_blocks(self, text): - """Ensure that Python interactive shell sessions are put in - code blocks -- even if not properly indented. - """ - if ">>>" not in text: - return text - - less_than_tab = self.tab_width - 1 - _pyshell_block_re = re.compile(r""" - ^([ ]{0,%d})>>>[ ].*\n # first line - ^(\1.*\S+.*\n)* # any number of subsequent lines - ^\n # ends with a blank line - """ % less_than_tab, re.M | re.X) - - return _pyshell_block_re.sub(self._pyshell_block_sub, text) - - def _run_span_gamut(self, text): - # These are all the transformations that occur *within* block-level - # tags like paragraphs, headers, and list items. - - text = self._do_code_spans(text) - - text = self._escape_special_chars(text) - - # Process anchor and image tags. - text = self._do_links(text) - - # Make links out of things like `` - # Must come after _do_links(), because you can use < and > - # delimiters in inline links like [this](). - text = self._do_auto_links(text) - - if "link-patterns" in self.extras: - text = self._do_link_patterns(text) - - text = self._encode_amps_and_angles(text) - - text = self._do_italics_and_bold(text) - - # Do hard breaks: - text = re.sub(r" {2,}\n", " - | - # auto-link (e.g., ) - <\w+[^>]*> - | - # comment - | - <\?.*?\?> # processing instruction - ) - """, re.X) - - def _escape_special_chars(self, text): - # Python markdown note: the HTML tokenization here differs from - # that in Markdown.pl, hence the behaviour for subtle cases can - # differ (I believe the tokenizer here does a better job because - # it isn't susceptible to unmatched '<' and '>' in HTML tags). - # Note, however, that '>' is not allowed in an auto-link URL - # here. - escaped = [] - is_html_markup = False - for token in self._sorta_html_tokenize_re.split(text): - if is_html_markup: - # Within tags/HTML-comments/auto-links, encode * and _ - # so they don't conflict with their use in Markdown for - # italics and strong. We're replacing each such - # character with its corresponding MD5 checksum value; - # this is likely overkill, but it should prevent us from - # colliding with the escape values by accident. - escaped.append(token.replace('*', g_escape_table['*']) - .replace('_', g_escape_table['_'])) - else: - escaped.append(self._encode_backslash_escapes(token)) - is_html_markup = not is_html_markup - return ''.join(escaped) - - def _hash_html_spans(self, text): - # Used for safe_mode. - - def _is_auto_link(s): - if ':' in s and self._auto_link_re.match(s): - return True - elif '@' in s and self._auto_email_link_re.match(s): - return True - return False - - tokens = [] - is_html_markup = False - for token in self._sorta_html_tokenize_re.split(text): - if is_html_markup and not _is_auto_link(token): - sanitized = self._sanitize_html(token) - key = _hash_text(sanitized) - self.html_spans[key] = sanitized - tokens.append(key) - else: - tokens.append(token) - is_html_markup = not is_html_markup - return ''.join(tokens) - - def _unhash_html_spans(self, text): - for key, sanitized in self.html_spans.items(): - text = text.replace(key, sanitized) - return text - - def _sanitize_html(self, s): - if self.safe_mode == "replace": - return self.html_removed_text - elif self.safe_mode == "escape": - replacements = [ - ('&', '&'), - ('<', '<'), - ('>', '>'), - ] - for before, after in replacements: - s = s.replace(before, after) - return s - else: - raise MarkdownError("invalid value for 'safe_mode': %r (must be " - "'escape' or 'replace')" % self.safe_mode) - - _tail_of_inline_link_re = re.compile(r''' - # Match tail of: [text](/url/) or [text](/url/ "title") - \( # literal paren - [ \t]* - (?P # \1 - <.*?> - | - .*? - ) - [ \t]* - ( # \2 - (['"]) # quote char = \3 - (?P.*?) - \3 # matching quote - )? # title is optional - \) - ''', re.X | re.S) - _tail_of_reference_link_re = re.compile(r''' - # Match tail of: [text][id] - [ ]? # one optional space - (?:\n[ ]*)? # one optional newline followed by spaces - \[ - (?P<id>.*?) - \] - ''', re.X | re.S) - - def _do_links(self, text): - """Turn Markdown link shortcuts into XHTML <a> and <img> tags. - - This is a combination of Markdown.pl's _DoAnchors() and - _DoImages(). They are done together because that simplified the - approach. It was necessary to use a different approach than - Markdown.pl because of the lack of atomic matching support in - Python's regex engine used in $g_nested_brackets. - """ - MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24 - - # `anchor_allowed_pos` is used to support img links inside - # anchors, but not anchors inside anchors. An anchor's start - # pos must be `>= anchor_allowed_pos`. - anchor_allowed_pos = 0 - - curr_pos = 0 - while True: # Handle the next link. - # The next '[' is the start of: - # - an inline anchor: [text](url "title") - # - a reference anchor: [text][id] - # - an inline img: ![text](url "title") - # - a reference img: ![text][id] - # - a footnote ref: [^id] - # (Only if 'footnotes' extra enabled) - # - a footnote defn: [^id]: ... - # (Only if 'footnotes' extra enabled) These have already - # been stripped in _strip_footnote_definitions() so no - # need to watch for them. - # - a link definition: [id]: url "title" - # These have already been stripped in - # _strip_link_definitions() so no need to watch for them. - # - not markup: [...anything else... - try: - start_idx = text.index('[', curr_pos) - except ValueError: - break - text_length = len(text) - - # Find the matching closing ']'. - # Markdown.pl allows *matching* brackets in link text so we - # will here too. Markdown.pl *doesn't* currently allow - # matching brackets in img alt text -- we'll differ in that - # regard. - bracket_depth = 0 - for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL, - text_length)): - ch = text[p] - if ch == ']': - bracket_depth -= 1 - if bracket_depth < 0: - break - elif ch == '[': - bracket_depth += 1 - else: - # Closing bracket not found within sentinel length. - # This isn't markup. - curr_pos = start_idx + 1 - continue - link_text = text[start_idx+1:p] - - # Possibly a footnote ref? - if "footnotes" in self.extras and link_text.startswith("^"): - normed_id = re.sub(r'\W', '-', link_text[1:]) - if normed_id in self.footnotes: - self.footnote_ids.append(normed_id) - result = '<sup class="footnote-ref" id="fnref-%s">' \ - '<a href="#fn-%s">%s</a></sup>' \ - % (normed_id, normed_id, len(self.footnote_ids)) - text = text[:start_idx] + result + text[p+1:] - else: - # This id isn't defined, leave the markup alone. - curr_pos = p+1 - continue - - # Now determine what this is by the remainder. - p += 1 - if p == text_length: - return text - - # Inline anchor or img? - if text[p] == '(': # attempt at perf improvement - match = self._tail_of_inline_link_re.match(text, p) - if match: - # Handle an inline anchor or img. - is_img = start_idx > 0 and text[start_idx-1] == "!" - if is_img: - start_idx -= 1 - - url, title = match.group("url"), match.group("title") - if url and url[0] == '<': - url = url[1:-1] # '<url>' -> 'url' - # We've got to encode these to avoid conflicting - # with italics/bold. - url = url.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - if title: - title_str = ' title="%s"' \ - % title.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) \ - .replace('"', '"') - else: - title_str = '' - if is_img: - result = '<img src="%s" alt="%s"%s%s' \ - % (url, link_text.replace('"', '"'), - title_str, self.empty_element_suffix) - curr_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - elif start_idx >= anchor_allowed_pos: - result_head = '<a href="%s"%s>' % (url, title_str) - result = '%s%s</a>' % (result_head, link_text) - # <img> allowed from curr_pos on, <a> from - # anchor_allowed_pos on. - curr_pos = start_idx + len(result_head) - anchor_allowed_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - else: - # Anchor not allowed here. - curr_pos = start_idx + 1 - continue - - # Reference anchor or img? - else: - match = self._tail_of_reference_link_re.match(text, p) - if match: - # Handle a reference-style anchor or img. - is_img = start_idx > 0 and text[start_idx-1] == "!" - if is_img: - start_idx -= 1 - link_id = match.group("id").lower() - if not link_id: - link_id = link_text.lower() # for links like [this][] - if link_id in self.urls: - url = self.urls[link_id] - # We've got to encode these to avoid conflicting - # with italics/bold. - url = url.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - title = self.titles.get(link_id) - if title: - title = title.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - title_str = ' title="%s"' % title - else: - title_str = '' - if is_img: - result = '<img src="%s" alt="%s"%s%s' \ - % (url, link_text.replace('"', '"'), - title_str, self.empty_element_suffix) - curr_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - elif start_idx >= anchor_allowed_pos: - result = '<a href="%s"%s>%s</a>' \ - % (url, title_str, link_text) - result_head = '<a href="%s"%s>' % (url, title_str) - result = '%s%s</a>' % (result_head, link_text) - # <img> allowed from curr_pos on, <a> from - # anchor_allowed_pos on. - curr_pos = start_idx + len(result_head) - anchor_allowed_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - else: - # Anchor not allowed here. - curr_pos = start_idx + 1 - else: - # This id isn't defined, leave the markup alone. - curr_pos = match.end() - continue - - # Otherwise, it isn't markup. - curr_pos = start_idx + 1 - - return text - - - _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M) - def _setext_h_sub(self, match): - n = {"=": 1, "-": 2}[match.group(2)[0]] - demote_headers = self.extras.get("demote-headers") - if demote_headers: - n = min(n + demote_headers, 6) - return "<h%d>%s</h%d>\n\n" \ - % (n, self._run_span_gamut(match.group(1)), n) - - _atx_h_re = re.compile(r''' - ^(\#{1,6}) # \1 = string of #'s - [ \t]* - (.+?) # \2 = Header text - [ \t]* - (?<!\\) # ensure not an escaped trailing '#' - \#* # optional closing #'s (not counted) - \n+ - ''', re.X | re.M) - def _atx_h_sub(self, match): - n = len(match.group(1)) - demote_headers = self.extras.get("demote-headers") - if demote_headers: - n = min(n + demote_headers, 6) - return "<h%d>%s</h%d>\n\n" \ - % (n, self._run_span_gamut(match.group(2)), n) - - def _do_headers(self, text): - # Setext-style headers: - # Header 1 - # ======== - # - # Header 2 - # -------- - text = self._setext_h_re.sub(self._setext_h_sub, text) - - # atx-style headers: - # # Header 1 - # ## Header 2 - # ## Header 2 with closing hashes ## - # ... - # ###### Header 6 - text = self._atx_h_re.sub(self._atx_h_sub, text) - - return text - - - _marker_ul_chars = '*+-' - _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars - _marker_ul = '(?:[%s])' % _marker_ul_chars - _marker_ol = r'(?:\d+\.)' - - def _list_sub(self, match): - lst = match.group(1) - lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol" - result = self._process_list_items(lst) - if self.list_level: - return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type) - else: - return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type) - - def _do_lists(self, text): - # Form HTML ordered (numbered) and unordered (bulleted) lists. - - for marker_pat in (self._marker_ul, self._marker_ol): - # Re-usable pattern to match any entire ul or ol list: - less_than_tab = self.tab_width - 1 - whole_list = r''' - ( # \1 = whole list - ( # \2 - [ ]{0,%d} - (%s) # \3 = first list item marker - [ \t]+ - ) - (?:.+?) - ( # \4 - \Z - | - \n{2,} - (?=\S) - (?! # Negative lookahead for another list item marker - [ \t]* - %s[ \t]+ - ) - ) - ) - ''' % (less_than_tab, marker_pat, marker_pat) - - # We use a different prefix before nested lists than top-level lists. - # See extended comment in _process_list_items(). - # - # Note: There's a bit of duplication here. My original implementation - # created a scalar regex pattern as the conditional result of the test on - # $g_list_level, and then only ran the $text =~ s{...}{...}egmx - # substitution once, using the scalar as the pattern. This worked, - # everywhere except when running under MT on my hosting account at Pair - # Networks. There, this caused all rebuilds to be killed by the reaper (or - # perhaps they crashed, but that seems incredibly unlikely given that the - # same script on the same server ran fine *except* under MT. I've spent - # more time trying to figure out why this is happening than I'd like to - # admit. My only guess, backed up by the fact that this workaround works, - # is that Perl optimizes the substition when it can figure out that the - # pattern will never change, and when this optimization isn't on, we run - # afoul of the reaper. Thus, the slightly redundant code to that uses two - # static s/// patterns rather than one conditional pattern. - - if self.list_level: - sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S) - text = sub_list_re.sub(self._list_sub, text) - else: - list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list, - re.X | re.M | re.S) - text = list_re.sub(self._list_sub, text) - - return text - - _list_item_re = re.compile(r''' - (\n)? # leading line = \1 - (^[ \t]*) # leading whitespace = \2 - (%s) [ \t]+ # list marker = \3 - ((?:.+?) # list item text = \4 - (\n{1,2})) # eols = \5 - (?= \n* (\Z | \2 (%s) [ \t]+)) - ''' % (_marker_any, _marker_any), - re.M | re.X | re.S) - - _last_li_endswith_two_eols = False - def _list_item_sub(self, match): - item = match.group(4) - leading_line = match.group(1) - leading_space = match.group(2) - if leading_line or "\n\n" in item or self._last_li_endswith_two_eols: - item = self._run_block_gamut(self._outdent(item)) - else: - # Recursion for sub-lists: - item = self._do_lists(self._outdent(item)) - if item.endswith('\n'): - item = item[:-1] - item = self._run_span_gamut(item) - self._last_li_endswith_two_eols = (len(match.group(5)) == 2) - return "<li>%s</li>\n" % item - - def _process_list_items(self, list_str): - # Process the contents of a single ordered or unordered list, - # splitting it into individual list items. - - # The $g_list_level global keeps track of when we're inside a list. - # Each time we enter a list, we increment it; when we leave a list, - # we decrement. If it's zero, we're not in a list anymore. - # - # We do this because when we're not inside a list, we want to treat - # something like this: - # - # I recommend upgrading to version - # 8. Oops, now this line is treated - # as a sub-list. - # - # As a single paragraph, despite the fact that the second line starts - # with a digit-period-space sequence. - # - # Whereas when we're inside a list (or sub-list), that line will be - # treated as the start of a sub-list. What a kludge, huh? This is - # an aspect of Markdown's syntax that's hard to parse perfectly - # without resorting to mind-reading. Perhaps the solution is to - # change the syntax rules such that sub-lists must start with a - # starting cardinal number; e.g. "1." or "a.". - self.list_level += 1 - self._last_li_endswith_two_eols = False - list_str = list_str.rstrip('\n') + '\n' - list_str = self._list_item_re.sub(self._list_item_sub, list_str) - self.list_level -= 1 - return list_str - - def _get_pygments_lexer(self, lexer_name): - try: - from pygments import lexers, util - except ImportError: - return None - try: - return lexers.get_lexer_by_name(lexer_name) - except util.ClassNotFound: - return None - - def _color_with_pygments(self, codeblock, lexer, **formatter_opts): - import pygments - import pygments.formatters - - class HtmlCodeFormatter(pygments.formatters.HtmlFormatter): - def _wrap_code(self, inner): - """A function for use in a Pygments Formatter which - wraps in <code> tags. - """ - yield 0, "<code>" - for tup in inner: - yield tup - yield 0, "</code>" - - def wrap(self, source, outfile): - """Return the source with a code, pre, and div.""" - return self._wrap_div(self._wrap_pre(self._wrap_code(source))) - - formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts) - return pygments.highlight(codeblock, lexer, formatter) - - def _code_block_sub(self, match): - codeblock = match.group(1) - codeblock = self._outdent(codeblock) - codeblock = self._detab(codeblock) - codeblock = codeblock.lstrip('\n') # trim leading newlines - codeblock = codeblock.rstrip() # trim trailing whitespace - - if "code-color" in self.extras and codeblock.startswith(":::"): - lexer_name, rest = codeblock.split('\n', 1) - lexer_name = lexer_name[3:].strip() - lexer = self._get_pygments_lexer(lexer_name) - codeblock = rest.lstrip("\n") # Remove lexer declaration line. - if lexer: - formatter_opts = self.extras['code-color'] or {} - colored = self._color_with_pygments(codeblock, lexer, - **formatter_opts) - return "\n\n%s\n\n" % colored - - codeblock = self._encode_code(codeblock) - return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock - - def _do_code_blocks(self, text): - """Process Markdown `<pre><code>` blocks.""" - code_block_re = re.compile(r''' - (?:\n\n|\A) - ( # $1 = the code block -- one or more lines, starting with a space/tab - (?: - (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces - .*\n+ - )+ - ) - ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc - ''' % (self.tab_width, self.tab_width), - re.M | re.X) - - return code_block_re.sub(self._code_block_sub, text) - - - # Rules for a code span: - # - backslash escapes are not interpreted in a code span - # - to include one or or a run of more backticks the delimiters must - # be a longer run of backticks - # - cannot start or end a code span with a backtick; pad with a - # space and that space will be removed in the emitted HTML - # See `test/tm-cases/escapes.text` for a number of edge-case - # examples. - _code_span_re = re.compile(r''' - (?<!\\) - (`+) # \1 = Opening run of ` - (?!`) # See Note A test/tm-cases/escapes.text - (.+?) # \2 = The code block - (?<!`) - \1 # Matching closer - (?!`) - ''', re.X | re.S) - - def _code_span_sub(self, match): - c = match.group(2).strip(" \t") - c = self._encode_code(c) - return "<code>%s</code>" % c - - def _do_code_spans(self, text): - # * Backtick quotes are used for <code></code> spans. - # - # * You can use multiple backticks as the delimiters if you want to - # include literal backticks in the code span. So, this input: - # - # Just type ``foo `bar` baz`` at the prompt. - # - # Will translate to: - # - # <p>Just type <code>foo `bar` baz</code> at the prompt.</p> - # - # There's no arbitrary limit to the number of backticks you - # can use as delimters. If you need three consecutive backticks - # in your code, use four for delimiters, etc. - # - # * You can use spaces to get literal backticks at the edges: - # - # ... type `` `bar` `` ... - # - # Turns to: - # - # ... type <code>`bar`</code> ... - return self._code_span_re.sub(self._code_span_sub, text) - - def _encode_code(self, text): - """Encode/escape certain characters inside Markdown code runs. - The point is that in code, these characters are literals, - and lose their special Markdown meanings. - """ - replacements = [ - # Encode all ampersands; HTML entities are not - # entities within a Markdown code span. - ('&', '&'), - # Do the angle bracket song and dance: - ('<', '<'), - ('>', '>'), - # Now, escape characters that are magic in Markdown: - ('*', g_escape_table['*']), - ('_', g_escape_table['_']), - ('{', g_escape_table['{']), - ('}', g_escape_table['}']), - ('[', g_escape_table['[']), - (']', g_escape_table[']']), - ('\\', g_escape_table['\\']), - ] - for before, after in replacements: - text = text.replace(before, after) - return text - - _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S) - _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S) - _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S) - _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S) - def _do_italics_and_bold(self, text): - # <strong> must go first: - if "code-friendly" in self.extras: - text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text) - text = self._code_friendly_em_re.sub(r"<em>\1</em>", text) - else: - text = self._strong_re.sub(r"<strong>\2</strong>", text) - text = self._em_re.sub(r"<em>\2</em>", text) - return text - - - _block_quote_re = re.compile(r''' - ( # Wrap whole match in \1 - ( - ^[ \t]*>[ \t]? # '>' at the start of a line - .+\n # rest of the first line - (.+\n)* # subsequent consecutive lines - \n* # blanks - )+ - ) - ''', re.M | re.X) - _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M); - - _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S) - def _dedent_two_spaces_sub(self, match): - return re.sub(r'(?m)^ ', '', match.group(1)) - - def _block_quote_sub(self, match): - bq = match.group(1) - bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting - bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines - bq = self._run_block_gamut(bq) # recurse - - bq = re.sub('(?m)^', ' ', bq) - # These leading spaces screw with <pre> content, so we need to fix that: - bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq) - - return "<blockquote>\n%s\n</blockquote>\n\n" % bq - - def _do_block_quotes(self, text): - if '>' not in text: - return text - return self._block_quote_re.sub(self._block_quote_sub, text) - - def _form_paragraphs(self, text): - # Strip leading and trailing lines: - text = text.strip('\n') - - # Wrap <p> tags. - grafs = re.split(r"\n{2,}", text) - for i, graf in enumerate(grafs): - if graf in self.html_blocks: - # Unhashify HTML blocks - grafs[i] = self.html_blocks[graf] - else: - # Wrap <p> tags. - graf = self._run_span_gamut(graf) - grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>" - - return "\n\n".join(grafs) - - def _add_footnotes(self, text): - if self.footnotes: - footer = [ - '<div class="footnotes">', - '<hr' + self.empty_element_suffix, - '<ol>', - ] - for i, id in enumerate(self.footnote_ids): - if i != 0: - footer.append('') - footer.append('<li id="fn-%s">' % id) - footer.append(self._run_block_gamut(self.footnotes[id])) - backlink = ('<a href="#fnref-%s" ' - 'class="footnoteBackLink" ' - 'title="Jump back to footnote %d in the text.">' - '↩</a>' % (id, i+1)) - if footer[-1].endswith("</p>"): - footer[-1] = footer[-1][:-len("</p>")] \ - + ' ' + backlink + "</p>" - else: - footer.append("\n<p>%s</p>" % backlink) - footer.append('</li>') - footer.append('</ol>') - footer.append('</div>') - return text + '\n\n' + '\n'.join(footer) - else: - return text - - # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin: - # http://bumppo.net/projects/amputator/ - _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)') - _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I) - _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I) - - def _encode_amps_and_angles(self, text): - # Smart processing for ampersands and angle brackets that need - # to be encoded. - text = self._ampersand_re.sub('&', text) - - # Encode naked <'s - text = self._naked_lt_re.sub('<', text) - - # Encode naked >'s - # Note: Other markdown implementations (e.g. Markdown.pl, PHP - # Markdown) don't do this. - text = self._naked_gt_re.sub('>', text) - return text - - def _encode_backslash_escapes(self, text): - for ch, escape in g_escape_table.items(): - text = text.replace("\\"+ch, escape) - return text - - _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I) - def _auto_link_sub(self, match): - g1 = match.group(1) - return '<a href="%s">%s</a>' % (g1, g1) - - _auto_email_link_re = re.compile(r""" - < - (?:mailto:)? - ( - [-.\w]+ - \@ - [-\w]+(\.[-\w]+)*\.[a-z]+ - ) - > - """, re.I | re.X | re.U) - def _auto_email_link_sub(self, match): - return self._encode_email_address( - self._unescape_special_chars(match.group(1))) - - def _do_auto_links(self, text): - text = self._auto_link_re.sub(self._auto_link_sub, text) - text = self._auto_email_link_re.sub(self._auto_email_link_sub, text) - return text - - def _encode_email_address(self, addr): - # Input: an email address, e.g. "foo@example.com" - # - # Output: the email address as a mailto link, with each character - # of the address encoded as either a decimal or hex entity, in - # the hopes of foiling most address harvesting spam bots. E.g.: - # - # <a href="mailto:foo@e - # xample.com">foo - # @example.com</a> - # - # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk - # mailing list: <http://tinyurl.com/yu7ue> - chars = [_xml_encode_email_char_at_random(ch) - for ch in "mailto:" + addr] - # Strip the mailto: from the visible part. - addr = '<a href="%s">%s</a>' \ - % (''.join(chars), ''.join(chars[7:])) - return addr - - def _do_link_patterns(self, text): - """Caveat emptor: there isn't much guarding against link - patterns being formed inside other standard Markdown links, e.g. - inside a [link def][like this]. - - Dev Notes: *Could* consider prefixing regexes with a negative - lookbehind assertion to attempt to guard against this. - """ - link_from_hash = {} - for regex, repl in self.link_patterns: - replacements = [] - for match in regex.finditer(text): - if hasattr(repl, "__call__"): - href = repl(match) - else: - href = match.expand(repl) - replacements.append((match.span(), href)) - for (start, end), href in reversed(replacements): - escaped_href = ( - href.replace('"', '"') # b/c of attr quote - # To avoid markdown <em> and <strong>: - .replace('*', g_escape_table['*']) - .replace('_', g_escape_table['_'])) - link = '<a href="%s">%s</a>' % (escaped_href, text[start:end]) - hash = md5(link).hexdigest() - link_from_hash[hash] = link - text = text[:start] + hash + text[end:] - for hash, link in link_from_hash.items(): - text = text.replace(hash, link) - return text - - def _unescape_special_chars(self, text): - # Swap back in all the special characters we've hidden. - for ch, hash in g_escape_table.items(): - text = text.replace(hash, ch) - return text - - def _outdent(self, text): - # Remove one level of line-leading tabs or spaces - return self._outdent_re.sub('', text) - - -class MarkdownWithExtras(Markdown): - """A markdowner class that enables most extras: - - - footnotes - - code-color (only has effect if 'pygments' Python module on path) - - These are not included: - - pyshell (specific to Python-related documenting) - - code-friendly (because it *disables* part of the syntax) - - link-patterns (because you need to specify some actual - link-patterns anyway) - """ - extras = ["footnotes", "code-color"] - - -#---- internal support functions - -# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549 -def _curry(*args, **kwargs): - function, args = args[0], args[1:] - def result(*rest, **kwrest): - combined = kwargs.copy() - combined.update(kwrest) - return function(*args + rest, **combined) - return result - -# Recipe: regex_from_encoded_pattern (1.0) -def _regex_from_encoded_pattern(s): - """'foo' -> re.compile(re.escape('foo')) - '/foo/' -> re.compile('foo') - '/foo/i' -> re.compile('foo', re.I) - """ - if s.startswith('/') and s.rfind('/') != 0: - # Parse it: /PATTERN/FLAGS - idx = s.rfind('/') - pattern, flags_str = s[1:idx], s[idx+1:] - flag_from_char = { - "i": re.IGNORECASE, - "l": re.LOCALE, - "s": re.DOTALL, - "m": re.MULTILINE, - "u": re.UNICODE, - } - flags = 0 - for char in flags_str: - try: - flags |= flag_from_char[char] - except KeyError: - raise ValueError("unsupported regex flag: '%s' in '%s' " - "(must be one of '%s')" - % (char, s, ''.join(flag_from_char.keys()))) - return re.compile(s[1:idx], flags) - else: # not an encoded regex - return re.compile(re.escape(s)) - -# Recipe: dedent (0.1.2) -def _dedentlines(lines, tabsize=8, skip_first_line=False): - """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines - - "lines" is a list of lines to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - Same as dedent() except operates on a sequence of lines. Note: the - lines list is modified **in-place**. - """ - DEBUG = False - if DEBUG: - print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ - % (tabsize, skip_first_line) - indents = [] - margin = None - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - indent = 0 - for ch in line: - if ch == ' ': - indent += 1 - elif ch == '\t': - indent += tabsize - (indent % tabsize) - elif ch in '\r\n': - continue # skip all-whitespace lines - else: - break - else: - continue # skip all-whitespace lines - if DEBUG: print "dedent: indent=%d: %r" % (indent, line) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if DEBUG: print "dedent: margin=%r" % margin - - if margin is not None and margin > 0: - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - removed = 0 - for j, ch in enumerate(line): - if ch == ' ': - removed += 1 - elif ch == '\t': - removed += tabsize - (removed % tabsize) - elif ch in '\r\n': - if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line - lines[i] = lines[i][j:] - break - else: - raise ValueError("unexpected non-whitespace char %r in " - "line %r while removing %d-space margin" - % (ch, line, margin)) - if DEBUG: - print "dedent: %r: %r -> removed %d/%d"\ - % (line, ch, removed, margin) - if removed == margin: - lines[i] = lines[i][j+1:] - break - elif removed > margin: - lines[i] = ' '*(removed-margin) + lines[i][j+1:] - break - else: - if removed: - lines[i] = lines[i][removed:] - return lines - -def _dedent(text, tabsize=8, skip_first_line=False): - """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text - - "text" is the text to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - textwrap.dedent(s), but don't expand tabs to spaces - """ - lines = text.splitlines(1) - _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) - return ''.join(lines) - - -class _memoized(object): - """Decorator that caches a function's return value each time it is called. - If called later with the same arguments, the cached value is returned, and - not re-evaluated. - - http://wiki.python.org/moin/PythonDecoratorLibrary - """ - def __init__(self, func): - self.func = func - self.cache = {} - def __call__(self, *args): - try: - return self.cache[args] - except KeyError: - self.cache[args] = value = self.func(*args) - return value - except TypeError: - # uncachable -- for instance, passing a list as an argument. - # Better to not cache than to blow up entirely. - return self.func(*args) - def __repr__(self): - """Return the function's docstring.""" - return self.func.__doc__ - - -def _xml_oneliner_re_from_tab_width(tab_width): - """Standalone XML processing instruction regex.""" - return re.compile(r""" - (?: - (?<=\n\n) # Starting after a blank line - | # or - \A\n? # the beginning of the doc - ) - ( # save in $1 - [ ]{0,%d} - (?: - <\?\w+\b\s+.*?\?> # XML processing instruction - | - <\w+:\w+\b\s+.*?/> # namespaced single tag - ) - [ \t]* - (?=\n{2,}|\Z) # followed by a blank line or end of document - ) - """ % (tab_width - 1), re.X) -_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width) - -def _hr_tag_re_from_tab_width(tab_width): - return re.compile(r""" - (?: - (?<=\n\n) # Starting after a blank line - | # or - \A\n? # the beginning of the doc - ) - ( # save in \1 - [ ]{0,%d} - <(hr) # start tag = \2 - \b # word break - ([^<>])*? # - /?> # the matching end tag - [ \t]* - (?=\n{2,}|\Z) # followed by a blank line or end of document - ) - """ % (tab_width - 1), re.X) -_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width) - - -def _xml_encode_email_char_at_random(ch): - r = random() - # Roughly 10% raw, 45% hex, 45% dec. - # '@' *must* be encoded. I [John Gruber] insist. - # Issue 26: '_' must be encoded. - if r > 0.9 and ch not in "@_": - return ch - elif r < 0.45: - # The [1:] is to drop leading '0': 0x63 -> x63 - return '&#%s;' % hex(ord(ch))[1:] - else: - return '&#%s;' % ord(ch) - -def _hash_text(text): - return 'md5:'+md5(text.encode("utf-8")).hexdigest() - - -#---- mainline - -class _NoReflowFormatter(optparse.IndentedHelpFormatter): - """An optparse formatter that does NOT reflow the description.""" - def format_description(self, description): - return description or "" - -def _test(): - import doctest - doctest.testmod() - -def main(argv=None): - if argv is None: - argv = sys.argv - if not logging.root.handlers: - logging.basicConfig() - - usage = "usage: %prog [PATHS...]" - version = "%prog "+__version__ - parser = optparse.OptionParser(prog="markdown2", usage=usage, - version=version, description=cmdln_desc, - formatter=_NoReflowFormatter()) - parser.add_option("-v", "--verbose", dest="log_level", - action="store_const", const=logging.DEBUG, - help="more verbose output") - parser.add_option("--encoding", - help="specify encoding of text content") - parser.add_option("--html4tags", action="store_true", default=False, - help="use HTML 4 style for empty element tags") - parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode", - help="sanitize literal HTML: 'escape' escapes " - "HTML meta chars, 'replace' replaces with an " - "[HTML_REMOVED] note") - parser.add_option("-x", "--extras", action="append", - help="Turn on specific extra features (not part of " - "the core Markdown spec). Supported values: " - "'code-friendly' disables _/__ for emphasis; " - "'code-color' adds code-block syntax coloring; " - "'link-patterns' adds auto-linking based on patterns; " - "'footnotes' adds the footnotes syntax;" - "'xml' passes one-liner processing instructions and namespaced XML tags;" - "'pyshell' to put unindented Python interactive shell sessions in a <code> block.") - parser.add_option("--use-file-vars", - help="Look for and use Emacs-style 'markdown-extras' " - "file var to turn on extras. See " - "<http://code.google.com/p/python-markdown2/wiki/Extras>.") - parser.add_option("--link-patterns-file", - help="path to a link pattern file") - parser.add_option("--self-test", action="store_true", - help="run internal self-tests (some doctests)") - parser.add_option("--compare", action="store_true", - help="run against Markdown.pl as well (for testing)") - parser.set_defaults(log_level=logging.INFO, compare=False, - encoding="utf-8", safe_mode=None, use_file_vars=False) - opts, paths = parser.parse_args() - log.setLevel(opts.log_level) - - if opts.self_test: - return _test() - - if opts.extras: - extras = {} - for s in opts.extras: - splitter = re.compile("[,;: ]+") - for e in splitter.split(s): - if '=' in e: - ename, earg = e.split('=', 1) - try: - earg = int(earg) - except ValueError: - pass - else: - ename, earg = e, None - extras[ename] = earg - else: - extras = None - - if opts.link_patterns_file: - link_patterns = [] - f = open(opts.link_patterns_file) - try: - for i, line in enumerate(f.readlines()): - if not line.strip(): continue - if line.lstrip().startswith("#"): continue - try: - pat, href = line.rstrip().rsplit(None, 1) - except ValueError: - raise MarkdownError("%s:%d: invalid link pattern line: %r" - % (opts.link_patterns_file, i+1, line)) - link_patterns.append( - (_regex_from_encoded_pattern(pat), href)) - finally: - f.close() - else: - link_patterns = None - - from os.path import join, dirname, abspath, exists - markdown_pl = join(dirname(dirname(abspath(__file__))), "test", - "Markdown.pl") - for path in paths: - if opts.compare: - print "==== Markdown.pl ====" - perl_cmd = 'perl %s "%s"' % (markdown_pl, path) - o = os.popen(perl_cmd) - perl_html = o.read() - o.close() - sys.stdout.write(perl_html) - print "==== markdown2.py ====" - html = markdown_path(path, encoding=opts.encoding, - html4tags=opts.html4tags, - safe_mode=opts.safe_mode, - extras=extras, link_patterns=link_patterns, - use_file_vars=opts.use_file_vars) - sys.stdout.write( - html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace')) - if opts.compare: - test_dir = join(dirname(dirname(abspath(__file__))), "test") - if exists(join(test_dir, "test_markdown2.py")): - sys.path.insert(0, test_dir) - from test_markdown2 import norm_html_from_html - norm_html = norm_html_from_html(html) - norm_perl_html = norm_html_from_html(perl_html) - else: - norm_html = html - norm_perl_html = perl_html - print "==== match? %r ====" % (norm_perl_html == norm_html) - - -if __name__ == "__main__": - sys.exit( main(sys.argv) ) - diff -Nru python-tornado-2.1.0/demos/appengine/static/blog.css python-tornado-3.1.1/demos/appengine/static/blog.css --- python-tornado-2.1.0/demos/appengine/static/blog.css 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/static/blog.css 2013-08-04 19:34:21.000000000 +0000 @@ -143,11 +143,11 @@ } .compose .title, -.compose .markdown { +.compose .body_source { width: 100%; } -.compose .markdown { +.compose .body_source { height: 500px; line-height: 16pt; } diff -Nru python-tornado-2.1.0/demos/appengine/templates/archive.html python-tornado-3.1.1/demos/appengine/templates/archive.html --- python-tornado-2.1.0/demos/appengine/templates/archive.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/archive.html 2013-08-04 19:34:21.000000000 +0000 @@ -23,7 +23,7 @@ <ul class="archive"> {% for entry in entries %} <li> - <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div> + <div class="title"><a href="/entry/{{ entry.slug }}">{{ entry.title }}</a></div> <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> </li> {% end %} diff -Nru python-tornado-2.1.0/demos/appengine/templates/base.html python-tornado-3.1.1/demos/appengine/templates/base.html --- python-tornado-2.1.0/demos/appengine/templates/base.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/base.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,10 +1,10 @@ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml"> <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> - <title>{{ escape(handler.settings["blog_title"]) }} + + {{ handler.settings["blog_title"] }} - + {% block head %}{% end %} @@ -12,15 +12,15 @@

{% block body %}{% end %}
diff -Nru python-tornado-2.1.0/demos/appengine/templates/compose.html python-tornado-3.1.1/demos/appengine/templates/compose.html --- python-tornado-2.1.0/demos/appengine/templates/compose.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/compose.html 2013-08-04 19:34:21.000000000 +0000 @@ -2,17 +2,16 @@ {% block body %}
-
-
+
+
{% if entry %} {% end %} - {{ xsrf_form_html() }} + {% module xsrf_form_html() %} {% end %} @@ -24,7 +23,7 @@ $(function() { $("input[name=title]").select(); $("form.compose").submit(function() { - var required = ["title", "markdown"]; + var required = ["title", "body_source"]; var form = $(this).get(0); for (var i = 0; i < required.length; i++) { if (!form[required[i]].value) { @@ -39,4 +38,3 @@ //]]> {% end %} - diff -Nru python-tornado-2.1.0/demos/appengine/templates/entry.html python-tornado-3.1.1/demos/appengine/templates/entry.html --- python-tornado-2.1.0/demos/appengine/templates/entry.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/entry.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,5 +1,5 @@ {% extends "base.html" %} {% block body %} - {{ modules.Entry(entry) }} + {% module Entry(entry) %} {% end %} diff -Nru python-tornado-2.1.0/demos/appengine/templates/feed.xml python-tornado-3.1.1/demos/appengine/templates/feed.xml --- python-tornado-2.1.0/demos/appengine/templates/feed.xml 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/feed.xml 2013-08-04 19:34:21.000000000 +0000 @@ -1,25 +1,25 @@ {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %} - {{ escape(handler.settings["blog_title"]) }} + {{ handler.settings["blog_title"] }} {% if len(entries) > 0 %} {{ max(e.updated for e in entries).strftime(date_format) }} {% else %} {{ datetime.datetime.utcnow().strftime(date_format) }} {% end %} http://{{ request.host }}/ - - - {{ escape(handler.settings["blog_title"]) }} + + + {{ handler.settings["blog_title"] }} {% for entry in entries %} http://{{ request.host }}/entry/{{ entry.slug }} - {{ escape(entry.title) }} + {{ entry.title }} {{ entry.updated.strftime(date_format) }} {{ entry.published.strftime(date_format) }} -
{{ entry.html }}
+
{% raw entry.html %}
 {% end %} diff -Nru python-tornado-2.1.0/demos/appengine/templates/home.html python-tornado-3.1.1/demos/appengine/templates/home.html --- python-tornado-2.1.0/demos/appengine/templates/home.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/home.html 2013-08-04 19:34:21.000000000 +0000 @@ -2,7 +2,7 @@ {% block body %} {% for entry in entries %} - {{ modules.Entry(entry) }} + {% module Entry(entry) %} {% end %} {% end %} diff -Nru python-tornado-2.1.0/demos/appengine/templates/modules/entry.html python-tornado-3.1.1/demos/appengine/templates/modules/entry.html --- python-tornado-2.1.0/demos/appengine/templates/modules/entry.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/appengine/templates/modules/entry.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,7 +1,7 @@
-

{{ escape(entry.title) }}

+

{{ entry.title }}

{{ locale.format_date(entry.published, full_format=True, shorter=True) }}
-
{{ entry.html }}
+
{% raw entry.html %}
{% if current_user and current_user.administrator %}
{{ _("Edit this post") }}
{% end %} diff -Nru python-tornado-2.1.0/demos/auth/authdemo.py python-tornado-3.1.1/demos/auth/authdemo.py --- python-tornado-2.1.0/demos/auth/authdemo.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/auth/authdemo.py 2013-08-04 19:34:21.000000000 +0000 @@ -18,10 +18,10 @@ import tornado.escape import tornado.httpserver import tornado.ioloop -import tornado.options import tornado.web -from tornado.options import define, options +from tornado import gen +from tornado.options import define, options, parse_command_line define("port", default=8888, help="run on the given port", type=int) @@ -34,7 +34,7 @@ (r"/auth/logout", LogoutHandler), ] settings = dict( - cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__", login_url="/auth/login", ) tornado.web.Application.__init__(self, handlers, **settings) @@ -42,7 +42,7 @@ class BaseHandler(tornado.web.RequestHandler): def get_current_user(self): - user_json = self.get_secure_cookie("user") + user_json = self.get_secure_cookie("authdemo_user") if not user_json: return None return tornado.escape.json_decode(user_json) @@ -57,25 +57,30 @@ class AuthHandler(BaseHandler, tornado.auth.GoogleMixin): @tornado.web.asynchronous + @gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) + user = yield self.get_authenticated_user() + self.set_secure_cookie("authdemo_user", + tornado.escape.json_encode(user)) + self.redirect("/") return self.authenticate_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - self.set_secure_cookie("user", tornado.escape.json_encode(user)) - self.redirect("/") + class LogoutHandler(BaseHandler): def get(self): - self.clear_cookie("user") - self.redirect("/") + # This logs the user out of this demo app, but does not log them + # out of Google. Since Google remembers previous authorizations, + # returning to this app will log them back in immediately with no + # interaction (unless they have separately logged out of Google in + # the meantime). + self.clear_cookie("authdemo_user") + self.write('You are now logged out. ' + 'Click here to log back in.') def main(): - tornado.options.parse_command_line() + parse_command_line() http_server = tornado.httpserver.HTTPServer(Application()) http_server.listen(options.port) tornado.ioloop.IOLoop.instance().start() diff -Nru python-tornado-2.1.0/demos/benchmark/benchmark.py python-tornado-3.1.1/demos/benchmark/benchmark.py --- python-tornado-2.1.0/demos/benchmark/benchmark.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/benchmark/benchmark.py 2013-08-04 19:34:21.000000000 +0000 @@ -39,6 +39,8 @@ # --n=15000 for its JIT to reach full effectiveness define("num_runs", type=int, default=1) +define("ioloop", type=str, default=None) + class RootHandler(RequestHandler): def get(self): self.write("Hello, world") @@ -51,6 +53,8 @@ def main(): parse_command_line() + if options.ioloop: + IOLoop.configure(options.ioloop) for i in xrange(options.num_runs): run() diff -Nru python-tornado-2.1.0/demos/benchmark/chunk_benchmark.py python-tornado-3.1.1/demos/benchmark/chunk_benchmark.py --- python-tornado-2.1.0/demos/benchmark/chunk_benchmark.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/demos/benchmark/chunk_benchmark.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Downloads a large file in chunked encoding with both curl and simple clients + +import logging +from tornado.curl_httpclient import CurlAsyncHTTPClient +from tornado.simple_httpclient import SimpleAsyncHTTPClient +from tornado.ioloop import IOLoop +from tornado.options import define, options, parse_command_line +from tornado.web import RequestHandler, Application + +define('port', default=8888) +define('num_chunks', default=1000) +define('chunk_size', default=2048) + +class ChunkHandler(RequestHandler): + def get(self): + for i in xrange(options.num_chunks): + self.write('A' * options.chunk_size) + self.flush() + self.finish() + +def main(): + parse_command_line() + app = Application([('/', ChunkHandler)]) + app.listen(options.port, address='127.0.0.1') + def callback(response): + response.rethrow() + assert len(response.body) == (options.num_chunks * options.chunk_size) + logging.warning("fetch completed in %s seconds", response.request_time) + IOLoop.instance().stop() + + logging.warning("Starting fetch with curl client") + curl_client = CurlAsyncHTTPClient() + curl_client.fetch('http://localhost:%d/' % options.port, + callback=callback) + IOLoop.instance().start() + + logging.warning("Starting fetch with simple client") + simple_client = SimpleAsyncHTTPClient() + simple_client.fetch('http://localhost:%d/' % options.port, + callback=callback) + IOLoop.instance().start() + + +if __name__ == '__main__': + main() diff -Nru python-tornado-2.1.0/demos/benchmark/stack_context_benchmark.py python-tornado-3.1.1/demos/benchmark/stack_context_benchmark.py --- python-tornado-2.1.0/demos/benchmark/stack_context_benchmark.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/demos/benchmark/stack_context_benchmark.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,75 @@ +#!/usr/bin/env python +"""Benchmark for stack_context functionality.""" +import collections +import contextlib +import functools +import subprocess +import sys + +from tornado import stack_context + +class Benchmark(object): + def enter_exit(self, count): + """Measures the overhead of the nested "with" statements + when using many contexts. + """ + if count < 0: + return + with self.make_context(): + self.enter_exit(count - 1) + + def call_wrapped(self, count): + """Wraps and calls a function at each level of stack depth + to measure the overhead of the wrapped function. + """ + # This queue is analogous to IOLoop.add_callback, but lets us + # benchmark the stack_context in isolation without system call + # overhead. + queue = collections.deque() + self.call_wrapped_inner(queue, count) + while queue: + queue.popleft()() + + def call_wrapped_inner(self, queue, count): + if count < 0: + return + with self.make_context(): + queue.append(stack_context.wrap( + functools.partial(self.call_wrapped_inner, queue, count - 1))) + +class StackBenchmark(Benchmark): + def make_context(self): + return stack_context.StackContext(self.__context) + + @contextlib.contextmanager + def __context(self): + yield + +class ExceptionBenchmark(Benchmark): + def make_context(self): + return stack_context.ExceptionStackContext(self.__handle_exception) + + def __handle_exception(self, typ, value, tb): + pass + +def main(): + base_cmd = [ + sys.executable, '-m', 'timeit', '-s', + 'from stack_context_benchmark import StackBenchmark, ExceptionBenchmark'] + cmds = [ + 'StackBenchmark().enter_exit(50)', + 'StackBenchmark().call_wrapped(50)', + 'StackBenchmark().enter_exit(500)', + 'StackBenchmark().call_wrapped(500)', + + 'ExceptionBenchmark().enter_exit(50)', + 'ExceptionBenchmark().call_wrapped(50)', + 'ExceptionBenchmark().enter_exit(500)', + 'ExceptionBenchmark().call_wrapped(500)', + ] + for cmd in cmds: + print cmd + subprocess.check_call(base_cmd + [cmd]) + +if __name__ == '__main__': + main() diff -Nru python-tornado-2.1.0/demos/blog/README python-tornado-3.1.1/demos/blog/README --- python-tornado-2.1.0/demos/blog/README 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/README 2013-08-04 19:34:21.000000000 +0000 @@ -16,6 +16,11 @@ can run "apt-get install mysql". Under OS X you can download the MySQL PKG file from http://dev.mysql.com/downloads/mysql/ +3. Install Python prerequisites + + Install the packages MySQL-python, torndb, and markdown (e.g. using pip or + easy_install) + 3. Connect to MySQL and create a database and user for the blog. Connect to MySQL as a user that can create databases and users: @@ -51,7 +56,7 @@ authentication. Currently the first user to connect will automatically be given the - ability to create and edit posts. + ability to create and edit posts. Once you've created one blog post, subsequent users will not be prompted to sign in. diff -Nru python-tornado-2.1.0/demos/blog/blog.py python-tornado-3.1.1/demos/blog/blog.py --- python-tornado-2.1.0/demos/blog/blog.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/blog.py 2013-08-04 19:34:21.000000000 +0000 @@ -17,8 +17,8 @@ import markdown import os.path import re +import torndb import tornado.auth -import tornado.database import tornado.httpserver import tornado.ioloop import tornado.options @@ -51,14 +51,14 @@ static_path=os.path.join(os.path.dirname(__file__), "static"), ui_modules={"Entry": EntryModule}, xsrf_cookies=True, - cookie_secret="11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__", login_url="/auth/login", - autoescape=None, + debug=True, ) tornado.web.Application.__init__(self, handlers, **settings) # Have one global connection to the blog DB across all handlers - self.db = tornado.database.Connection( + self.db = torndb.Connection( host=options.mysql_host, database=options.mysql_database, user=options.mysql_user, password=options.mysql_password) @@ -69,7 +69,7 @@ return self.application.db def get_current_user(self): - user_id = self.get_secure_cookie("user") + user_id = self.get_secure_cookie("blogdemo_user") if not user_id: return None return self.db.get("SELECT * FROM authors WHERE id = %s", int(user_id)) @@ -152,7 +152,7 @@ self.get_authenticated_user(self.async_callback(self._on_auth)) return self.authenticate_redirect() - + def _on_auth(self, user): if not user: raise tornado.web.HTTPError(500, "Google auth failed") @@ -170,13 +170,13 @@ return else: author_id = author["id"] - self.set_secure_cookie("user", str(author_id)) + self.set_secure_cookie("blogdemo_user", str(author_id)) self.redirect(self.get_argument("next", "/")) class AuthLogoutHandler(BaseHandler): def get(self): - self.clear_cookie("user") + self.clear_cookie("blogdemo_user") self.redirect(self.get_argument("next", "/")) diff -Nru python-tornado-2.1.0/demos/blog/markdown.py python-tornado-3.1.1/demos/blog/markdown.py --- python-tornado-2.1.0/demos/blog/markdown.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/markdown.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,1877 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2007-2008 ActiveState Corp. -# License: MIT (http://www.opensource.org/licenses/mit-license.php) - -r"""A fast and complete Python implementation of Markdown. - -[from http://daringfireball.net/projects/markdown/] -> Markdown is a text-to-HTML filter; it translates an easy-to-read / -> easy-to-write structured text format into HTML. Markdown's text -> format is most similar to that of plain text email, and supports -> features such as headers, *emphasis*, code blocks, blockquotes, and -> links. -> -> Markdown's syntax is designed not as a generic markup language, but -> specifically to serve as a front-end to (X)HTML. You can use span-level -> HTML tags anywhere in a Markdown document, and you can use block level -> HTML tags (like
and
as well). - -Module usage: - - >>> import markdown2 - >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)` - u'

boo!

\n' - - >>> markdowner = Markdown() - >>> markdowner.convert("*boo!*") - u'

boo!

\n' - >>> markdowner.convert("**boom!**") - u'

boom!

\n' - -This implementation of Markdown implements the full "core" syntax plus a -number of extras (e.g., code syntax coloring, footnotes) as described on -. -""" - -cmdln_desc = """A fast and complete Python implementation of Markdown, a -text-to-HTML conversion tool for web writers. -""" - -# Dev Notes: -# - There is already a Python markdown processor -# (http://www.freewisdom.org/projects/python-markdown/). -# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm -# not yet sure if there implications with this. Compare 'pydoc sre' -# and 'perldoc perlre'. - -__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl -__version__ = '1.0.1.14' -__author__ = "Trent Mick" - -import os -import sys -from pprint import pprint -import re -import logging -try: - from hashlib import md5 -except ImportError: - from md5 import md5 -import optparse -from random import random -import codecs - - - -#---- Python version compat - -if sys.version_info[:2] < (2,4): - from sets import Set as set - def reversed(sequence): - for i in sequence[::-1]: - yield i - def _unicode_decode(s, encoding, errors='xmlcharrefreplace'): - return unicode(s, encoding, errors) -else: - def _unicode_decode(s, encoding, errors='strict'): - return s.decode(encoding, errors) - - -#---- globals - -DEBUG = False -log = logging.getLogger("markdown") - -DEFAULT_TAB_WIDTH = 4 - -# Table of hash values for escaped characters: -def _escape_hash(s): - # Lame attempt to avoid possible collision with someone actually - # using the MD5 hexdigest of one of these chars in there text. - # Other ideas: random.random(), uuid.uuid() - #return md5(s).hexdigest() # Markdown.pl effectively does this. - return 'md5-'+md5(s).hexdigest() -g_escape_table = dict([(ch, _escape_hash(ch)) - for ch in '\\`*_{}[]()>#+-.!']) - - - -#---- exceptions - -class MarkdownError(Exception): - pass - - - -#---- public api - -def markdown_path(path, encoding="utf-8", - html4tags=False, tab_width=DEFAULT_TAB_WIDTH, - safe_mode=None, extras=None, link_patterns=None, - use_file_vars=False): - text = codecs.open(path, 'r', encoding).read() - return Markdown(html4tags=html4tags, tab_width=tab_width, - safe_mode=safe_mode, extras=extras, - link_patterns=link_patterns, - use_file_vars=use_file_vars).convert(text) - -def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH, - safe_mode=None, extras=None, link_patterns=None, - use_file_vars=False): - return Markdown(html4tags=html4tags, tab_width=tab_width, - safe_mode=safe_mode, extras=extras, - link_patterns=link_patterns, - use_file_vars=use_file_vars).convert(text) - -class Markdown(object): - # The dict of "extras" to enable in processing -- a mapping of - # extra name to argument for the extra. Most extras do not have an - # argument, in which case the value is None. - # - # This can be set via (a) subclassing and (b) the constructor - # "extras" argument. - extras = None - - urls = None - titles = None - html_blocks = None - html_spans = None - html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py - - # Used to track when we're inside an ordered or unordered list - # (see _ProcessListItems() for details): - list_level = 0 - - _ws_only_line_re = re.compile(r"^[ \t]+$", re.M) - - def __init__(self, html4tags=False, tab_width=4, safe_mode=None, - extras=None, link_patterns=None, use_file_vars=False): - if html4tags: - self.empty_element_suffix = ">" - else: - self.empty_element_suffix = " />" - self.tab_width = tab_width - - # For compatibility with earlier markdown2.py and with - # markdown.py's safe_mode being a boolean, - # safe_mode == True -> "replace" - if safe_mode is True: - self.safe_mode = "replace" - else: - self.safe_mode = safe_mode - - if self.extras is None: - self.extras = {} - elif not isinstance(self.extras, dict): - self.extras = dict([(e, None) for e in self.extras]) - if extras: - if not isinstance(extras, dict): - extras = dict([(e, None) for e in extras]) - self.extras.update(extras) - assert isinstance(self.extras, dict) - self._instance_extras = self.extras.copy() - self.link_patterns = link_patterns - self.use_file_vars = use_file_vars - self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M) - - def reset(self): - self.urls = {} - self.titles = {} - self.html_blocks = {} - self.html_spans = {} - self.list_level = 0 - self.extras = self._instance_extras.copy() - if "footnotes" in self.extras: - self.footnotes = {} - self.footnote_ids = [] - - def convert(self, text): - """Convert the given text.""" - # Main function. The order in which other subs are called here is - # essential. Link and image substitutions need to happen before - # _EscapeSpecialChars(), so that any *'s or _'s in the - # and tags get encoded. - - # Clear the global hashes. If we don't clear these, you get conflicts - # from other articles when generating a page which contains more than - # one article (e.g. an index page that shows the N most recent - # articles): - self.reset() - - if not isinstance(text, unicode): - #TODO: perhaps shouldn't presume UTF-8 for string input? - text = unicode(text, 'utf-8') - - if self.use_file_vars: - # Look for emacs-style file variable hints. - emacs_vars = self._get_emacs_vars(text) - if "markdown-extras" in emacs_vars: - splitter = re.compile("[ ,]+") - for e in splitter.split(emacs_vars["markdown-extras"]): - if '=' in e: - ename, earg = e.split('=', 1) - try: - earg = int(earg) - except ValueError: - pass - else: - ename, earg = e, None - self.extras[ename] = earg - - # Standardize line endings: - text = re.sub("\r\n|\r", "\n", text) - - # Make sure $text ends with a couple of newlines: - text += "\n\n" - - # Convert all tabs to spaces. - text = self._detab(text) - - # Strip any lines consisting only of spaces and tabs. - # This makes subsequent regexen easier to write, because we can - # match consecutive blank lines with /\n+/ instead of something - # contorted like /[ \t]*\n+/ . - text = self._ws_only_line_re.sub("", text) - - if self.safe_mode: - text = self._hash_html_spans(text) - - # Turn block-level HTML blocks into hash entries - text = self._hash_html_blocks(text, raw=True) - - # Strip link definitions, store in hashes. - if "footnotes" in self.extras: - # Must do footnotes first because an unlucky footnote defn - # looks like a link defn: - # [^4]: this "looks like a link defn" - text = self._strip_footnote_definitions(text) - text = self._strip_link_definitions(text) - - text = self._run_block_gamut(text) - - if "footnotes" in self.extras: - text = self._add_footnotes(text) - - text = self._unescape_special_chars(text) - - if self.safe_mode: - text = self._unhash_html_spans(text) - - text += "\n" - return text - - _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE) - # This regular expression is intended to match blocks like this: - # PREFIX Local Variables: SUFFIX - # PREFIX mode: Tcl SUFFIX - # PREFIX End: SUFFIX - # Some notes: - # - "[ \t]" is used instead of "\s" to specifically exclude newlines - # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does - # not like anything other than Unix-style line terminators. - _emacs_local_vars_pat = re.compile(r"""^ - (?P(?:[^\r\n|\n|\r])*?) - [\ \t]*Local\ Variables:[\ \t]* - (?P.*?)(?:\r\n|\n|\r) - (?P.*?\1End:) - """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE) - - def _get_emacs_vars(self, text): - """Return a dictionary of emacs-style local variables. - - Parsing is done loosely according to this spec (and according to - some in-practice deviations from this): - http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables - """ - emacs_vars = {} - SIZE = pow(2, 13) # 8kB - - # Search near the start for a '-*-'-style one-liner of variables. - head = text[:SIZE] - if "-*-" in head: - match = self._emacs_oneliner_vars_pat.search(head) - if match: - emacs_vars_str = match.group(1) - assert '\n' not in emacs_vars_str - emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';') - if s.strip()] - if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]: - # While not in the spec, this form is allowed by emacs: - # -*- Tcl -*- - # where the implied "variable" is "mode". This form - # is only allowed if there are no other variables. - emacs_vars["mode"] = emacs_var_strs[0].strip() - else: - for emacs_var_str in emacs_var_strs: - try: - variable, value = emacs_var_str.strip().split(':', 1) - except ValueError: - log.debug("emacs variables error: malformed -*- " - "line: %r", emacs_var_str) - continue - # Lowercase the variable name because Emacs allows "Mode" - # or "mode" or "MoDe", etc. - emacs_vars[variable.lower()] = value.strip() - - tail = text[-SIZE:] - if "Local Variables" in tail: - match = self._emacs_local_vars_pat.search(tail) - if match: - prefix = match.group("prefix") - suffix = match.group("suffix") - lines = match.group("content").splitlines(0) - #print "prefix=%r, suffix=%r, content=%r, lines: %s"\ - # % (prefix, suffix, match.group("content"), lines) - - # Validate the Local Variables block: proper prefix and suffix - # usage. - for i, line in enumerate(lines): - if not line.startswith(prefix): - log.debug("emacs variables error: line '%s' " - "does not use proper prefix '%s'" - % (line, prefix)) - return {} - # Don't validate suffix on last line. Emacs doesn't care, - # neither should we. - if i != len(lines)-1 and not line.endswith(suffix): - log.debug("emacs variables error: line '%s' " - "does not use proper suffix '%s'" - % (line, suffix)) - return {} - - # Parse out one emacs var per line. - continued_for = None - for line in lines[:-1]: # no var on the last line ("PREFIX End:") - if prefix: line = line[len(prefix):] # strip prefix - if suffix: line = line[:-len(suffix)] # strip suffix - line = line.strip() - if continued_for: - variable = continued_for - if line.endswith('\\'): - line = line[:-1].rstrip() - else: - continued_for = None - emacs_vars[variable] += ' ' + line - else: - try: - variable, value = line.split(':', 1) - except ValueError: - log.debug("local variables error: missing colon " - "in local variables entry: '%s'" % line) - continue - # Do NOT lowercase the variable name, because Emacs only - # allows "mode" (and not "Mode", "MoDe", etc.) in this block. - value = value.strip() - if value.endswith('\\'): - value = value[:-1].rstrip() - continued_for = variable - else: - continued_for = None - emacs_vars[variable] = value - - # Unquote values. - for var, val in emacs_vars.items(): - if len(val) > 1 and (val.startswith('"') and val.endswith('"') - or val.startswith('"') and val.endswith('"')): - emacs_vars[var] = val[1:-1] - - return emacs_vars - - # Cribbed from a post by Bart Lateur: - # - _detab_re = re.compile(r'(.*?)\t', re.M) - def _detab_sub(self, match): - g1 = match.group(1) - return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width)) - def _detab(self, text): - r"""Remove (leading?) tabs from a file. - - >>> m = Markdown() - >>> m._detab("\tfoo") - ' foo' - >>> m._detab(" \tfoo") - ' foo' - >>> m._detab("\t foo") - ' foo' - >>> m._detab(" foo") - ' foo' - >>> m._detab(" foo\n\tbar\tblam") - ' foo\n bar blam' - """ - if '\t' not in text: - return text - return self._detab_re.subn(self._detab_sub, text)[0] - - _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del' - _strict_tag_block_re = re.compile(r""" - ( # save in \1 - ^ # start of line (with re.M) - <(%s) # start tag = \2 - \b # word break - (.*\n)*? # any number of lines, minimally matching - # the matching end tag - [ \t]* # trailing spaces/tabs - (?=\n+|\Z) # followed by a newline or end of document - ) - """ % _block_tags_a, - re.X | re.M) - - _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math' - _liberal_tag_block_re = re.compile(r""" - ( # save in \1 - ^ # start of line (with re.M) - <(%s) # start tag = \2 - \b # word break - (.*\n)*? # any number of lines, minimally matching - .* # the matching end tag - [ \t]* # trailing spaces/tabs - (?=\n+|\Z) # followed by a newline or end of document - ) - """ % _block_tags_b, - re.X | re.M) - - def _hash_html_block_sub(self, match, raw=False): - html = match.group(1) - if raw and self.safe_mode: - html = self._sanitize_html(html) - key = _hash_text(html) - self.html_blocks[key] = html - return "\n\n" + key + "\n\n" - - def _hash_html_blocks(self, text, raw=False): - """Hashify HTML blocks - - We only want to do this for block-level HTML tags, such as headers, - lists, and tables. That's because we still want to wrap

s around - "paragraphs" that are wrapped in non-block-level tags, such as anchors, - phrase emphasis, and spans. The list of tags we're looking for is - hard-coded. - - @param raw {boolean} indicates if these are raw HTML blocks in - the original source. It makes a difference in "safe" mode. - """ - if '<' not in text: - return text - - # Pass `raw` value into our calls to self._hash_html_block_sub. - hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw) - - # First, look for nested blocks, e.g.: - #

- #
- # tags for inner block must be indented. - #
- #
- # - # The outermost tags must start at the left margin for this to match, and - # the inner nested divs must be indented. - # We need to do this before the next, more liberal match, because the next - # match will start at the first `
` and stop at the first `
`. - text = self._strict_tag_block_re.sub(hash_html_block_sub, text) - - # Now match more liberally, simply from `\n` to `\n` - text = self._liberal_tag_block_re.sub(hash_html_block_sub, text) - - # Special case just for
. It was easier to make a special - # case than to make the other regex more complicated. - if "", start_idx) + 3 - except ValueError, ex: - break - - # Start position for next comment block search. - start = end_idx - - # Validate whitespace before comment. - if start_idx: - # - Up to `tab_width - 1` spaces before start_idx. - for i in range(self.tab_width - 1): - if text[start_idx - 1] != ' ': - break - start_idx -= 1 - if start_idx == 0: - break - # - Must be preceded by 2 newlines or hit the start of - # the document. - if start_idx == 0: - pass - elif start_idx == 1 and text[0] == '\n': - start_idx = 0 # to match minute detail of Markdown.pl regex - elif text[start_idx-2:start_idx] == '\n\n': - pass - else: - break - - # Validate whitespace after comment. - # - Any number of spaces and tabs. - while end_idx < len(text): - if text[end_idx] not in ' \t': - break - end_idx += 1 - # - Must be following by 2 newlines or hit end of text. - if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'): - continue - - # Escape and hash (must match `_hash_html_block_sub`). - html = text[start_idx:end_idx] - if raw and self.safe_mode: - html = self._sanitize_html(html) - key = _hash_text(html) - self.html_blocks[key] = html - text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:] - - if "xml" in self.extras: - # Treat XML processing instructions and namespaced one-liner - # tags as if they were block HTML tags. E.g., if standalone - # (i.e. are their own paragraph), the following do not get - # wrapped in a

tag: - # - # - # - _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width) - text = _xml_oneliner_re.sub(hash_html_block_sub, text) - - return text - - def _strip_link_definitions(self, text): - # Strips link definitions from text, stores the URLs and titles in - # hash references. - less_than_tab = self.tab_width - 1 - - # Link defs are in the form: - # [id]: url "optional title" - _link_def_re = re.compile(r""" - ^[ ]{0,%d}\[(.+)\]: # id = \1 - [ \t]* - \n? # maybe *one* newline - [ \t]* - ? # url = \2 - [ \t]* - (?: - \n? # maybe one newline - [ \t]* - (?<=\s) # lookbehind for whitespace - ['"(] - ([^\n]*) # title = \3 - ['")] - [ \t]* - )? # title is optional - (?:\n+|\Z) - """ % less_than_tab, re.X | re.M | re.U) - return _link_def_re.sub(self._extract_link_def_sub, text) - - def _extract_link_def_sub(self, match): - id, url, title = match.groups() - key = id.lower() # Link IDs are case-insensitive - self.urls[key] = self._encode_amps_and_angles(url) - if title: - self.titles[key] = title.replace('"', '"') - return "" - - def _extract_footnote_def_sub(self, match): - id, text = match.groups() - text = _dedent(text, skip_first_line=not text.startswith('\n')).strip() - normed_id = re.sub(r'\W', '-', id) - # Ensure footnote text ends with a couple newlines (for some - # block gamut matches). - self.footnotes[normed_id] = text + "\n\n" - return "" - - def _strip_footnote_definitions(self, text): - """A footnote definition looks like this: - - [^note-id]: Text of the note. - - May include one or more indented paragraphs. - - Where, - - The 'note-id' can be pretty much anything, though typically it - is the number of the footnote. - - The first paragraph may start on the next line, like so: - - [^note-id]: - Text of the note. - """ - less_than_tab = self.tab_width - 1 - footnote_def_re = re.compile(r''' - ^[ ]{0,%d}\[\^(.+)\]: # id = \1 - [ \t]* - ( # footnote text = \2 - # First line need not start with the spaces. - (?:\s*.*\n+) - (?: - (?:[ ]{%d} | \t) # Subsequent lines must be indented. - .*\n+ - )* - ) - # Lookahead for non-space at line-start, or end of doc. - (?:(?=^[ ]{0,%d}\S)|\Z) - ''' % (less_than_tab, self.tab_width, self.tab_width), - re.X | re.M) - return footnote_def_re.sub(self._extract_footnote_def_sub, text) - - - _hr_res = [ - re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M), - re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M), - re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M), - ] - - def _run_block_gamut(self, text): - # These are all the transformations that form block-level - # tags like paragraphs, headers, and list items. - - text = self._do_headers(text) - - # Do Horizontal Rules: - hr = "\n tags around block-level tags. - text = self._hash_html_blocks(text) - - text = self._form_paragraphs(text) - - return text - - def _pyshell_block_sub(self, match): - lines = match.group(0).splitlines(0) - _dedentlines(lines) - indent = ' ' * self.tab_width - s = ('\n' # separate from possible cuddled paragraph - + indent + ('\n'+indent).join(lines) - + '\n\n') - return s - - def _prepare_pyshell_blocks(self, text): - """Ensure that Python interactive shell sessions are put in - code blocks -- even if not properly indented. - """ - if ">>>" not in text: - return text - - less_than_tab = self.tab_width - 1 - _pyshell_block_re = re.compile(r""" - ^([ ]{0,%d})>>>[ ].*\n # first line - ^(\1.*\S+.*\n)* # any number of subsequent lines - ^\n # ends with a blank line - """ % less_than_tab, re.M | re.X) - - return _pyshell_block_re.sub(self._pyshell_block_sub, text) - - def _run_span_gamut(self, text): - # These are all the transformations that occur *within* block-level - # tags like paragraphs, headers, and list items. - - text = self._do_code_spans(text) - - text = self._escape_special_chars(text) - - # Process anchor and image tags. - text = self._do_links(text) - - # Make links out of things like `` - # Must come after _do_links(), because you can use < and > - # delimiters in inline links like [this](). - text = self._do_auto_links(text) - - if "link-patterns" in self.extras: - text = self._do_link_patterns(text) - - text = self._encode_amps_and_angles(text) - - text = self._do_italics_and_bold(text) - - # Do hard breaks: - text = re.sub(r" {2,}\n", " - | - # auto-link (e.g., ) - <\w+[^>]*> - | - # comment - | - <\?.*?\?> # processing instruction - ) - """, re.X) - - def _escape_special_chars(self, text): - # Python markdown note: the HTML tokenization here differs from - # that in Markdown.pl, hence the behaviour for subtle cases can - # differ (I believe the tokenizer here does a better job because - # it isn't susceptible to unmatched '<' and '>' in HTML tags). - # Note, however, that '>' is not allowed in an auto-link URL - # here. - escaped = [] - is_html_markup = False - for token in self._sorta_html_tokenize_re.split(text): - if is_html_markup: - # Within tags/HTML-comments/auto-links, encode * and _ - # so they don't conflict with their use in Markdown for - # italics and strong. We're replacing each such - # character with its corresponding MD5 checksum value; - # this is likely overkill, but it should prevent us from - # colliding with the escape values by accident. - escaped.append(token.replace('*', g_escape_table['*']) - .replace('_', g_escape_table['_'])) - else: - escaped.append(self._encode_backslash_escapes(token)) - is_html_markup = not is_html_markup - return ''.join(escaped) - - def _hash_html_spans(self, text): - # Used for safe_mode. - - def _is_auto_link(s): - if ':' in s and self._auto_link_re.match(s): - return True - elif '@' in s and self._auto_email_link_re.match(s): - return True - return False - - tokens = [] - is_html_markup = False - for token in self._sorta_html_tokenize_re.split(text): - if is_html_markup and not _is_auto_link(token): - sanitized = self._sanitize_html(token) - key = _hash_text(sanitized) - self.html_spans[key] = sanitized - tokens.append(key) - else: - tokens.append(token) - is_html_markup = not is_html_markup - return ''.join(tokens) - - def _unhash_html_spans(self, text): - for key, sanitized in self.html_spans.items(): - text = text.replace(key, sanitized) - return text - - def _sanitize_html(self, s): - if self.safe_mode == "replace": - return self.html_removed_text - elif self.safe_mode == "escape": - replacements = [ - ('&', '&'), - ('<', '<'), - ('>', '>'), - ] - for before, after in replacements: - s = s.replace(before, after) - return s - else: - raise MarkdownError("invalid value for 'safe_mode': %r (must be " - "'escape' or 'replace')" % self.safe_mode) - - _tail_of_inline_link_re = re.compile(r''' - # Match tail of: [text](/url/) or [text](/url/ "title") - \( # literal paren - [ \t]* - (?P # \1 - <.*?> - | - .*? - ) - [ \t]* - ( # \2 - (['"]) # quote char = \3 - (?P.*?) - \3 # matching quote - )? # title is optional - \) - ''', re.X | re.S) - _tail_of_reference_link_re = re.compile(r''' - # Match tail of: [text][id] - [ ]? # one optional space - (?:\n[ ]*)? # one optional newline followed by spaces - \[ - (?P<id>.*?) - \] - ''', re.X | re.S) - - def _do_links(self, text): - """Turn Markdown link shortcuts into XHTML <a> and <img> tags. - - This is a combination of Markdown.pl's _DoAnchors() and - _DoImages(). They are done together because that simplified the - approach. It was necessary to use a different approach than - Markdown.pl because of the lack of atomic matching support in - Python's regex engine used in $g_nested_brackets. - """ - MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24 - - # `anchor_allowed_pos` is used to support img links inside - # anchors, but not anchors inside anchors. An anchor's start - # pos must be `>= anchor_allowed_pos`. - anchor_allowed_pos = 0 - - curr_pos = 0 - while True: # Handle the next link. - # The next '[' is the start of: - # - an inline anchor: [text](url "title") - # - a reference anchor: [text][id] - # - an inline img: ![text](url "title") - # - a reference img: ![text][id] - # - a footnote ref: [^id] - # (Only if 'footnotes' extra enabled) - # - a footnote defn: [^id]: ... - # (Only if 'footnotes' extra enabled) These have already - # been stripped in _strip_footnote_definitions() so no - # need to watch for them. - # - a link definition: [id]: url "title" - # These have already been stripped in - # _strip_link_definitions() so no need to watch for them. - # - not markup: [...anything else... - try: - start_idx = text.index('[', curr_pos) - except ValueError: - break - text_length = len(text) - - # Find the matching closing ']'. - # Markdown.pl allows *matching* brackets in link text so we - # will here too. Markdown.pl *doesn't* currently allow - # matching brackets in img alt text -- we'll differ in that - # regard. - bracket_depth = 0 - for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL, - text_length)): - ch = text[p] - if ch == ']': - bracket_depth -= 1 - if bracket_depth < 0: - break - elif ch == '[': - bracket_depth += 1 - else: - # Closing bracket not found within sentinel length. - # This isn't markup. - curr_pos = start_idx + 1 - continue - link_text = text[start_idx+1:p] - - # Possibly a footnote ref? - if "footnotes" in self.extras and link_text.startswith("^"): - normed_id = re.sub(r'\W', '-', link_text[1:]) - if normed_id in self.footnotes: - self.footnote_ids.append(normed_id) - result = '<sup class="footnote-ref" id="fnref-%s">' \ - '<a href="#fn-%s">%s</a></sup>' \ - % (normed_id, normed_id, len(self.footnote_ids)) - text = text[:start_idx] + result + text[p+1:] - else: - # This id isn't defined, leave the markup alone. - curr_pos = p+1 - continue - - # Now determine what this is by the remainder. - p += 1 - if p == text_length: - return text - - # Inline anchor or img? - if text[p] == '(': # attempt at perf improvement - match = self._tail_of_inline_link_re.match(text, p) - if match: - # Handle an inline anchor or img. - is_img = start_idx > 0 and text[start_idx-1] == "!" - if is_img: - start_idx -= 1 - - url, title = match.group("url"), match.group("title") - if url and url[0] == '<': - url = url[1:-1] # '<url>' -> 'url' - # We've got to encode these to avoid conflicting - # with italics/bold. - url = url.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - if title: - title_str = ' title="%s"' \ - % title.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) \ - .replace('"', '"') - else: - title_str = '' - if is_img: - result = '<img src="%s" alt="%s"%s%s' \ - % (url, link_text.replace('"', '"'), - title_str, self.empty_element_suffix) - curr_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - elif start_idx >= anchor_allowed_pos: - result_head = '<a href="%s"%s>' % (url, title_str) - result = '%s%s</a>' % (result_head, link_text) - # <img> allowed from curr_pos on, <a> from - # anchor_allowed_pos on. - curr_pos = start_idx + len(result_head) - anchor_allowed_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - else: - # Anchor not allowed here. - curr_pos = start_idx + 1 - continue - - # Reference anchor or img? - else: - match = self._tail_of_reference_link_re.match(text, p) - if match: - # Handle a reference-style anchor or img. - is_img = start_idx > 0 and text[start_idx-1] == "!" - if is_img: - start_idx -= 1 - link_id = match.group("id").lower() - if not link_id: - link_id = link_text.lower() # for links like [this][] - if link_id in self.urls: - url = self.urls[link_id] - # We've got to encode these to avoid conflicting - # with italics/bold. - url = url.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - title = self.titles.get(link_id) - if title: - title = title.replace('*', g_escape_table['*']) \ - .replace('_', g_escape_table['_']) - title_str = ' title="%s"' % title - else: - title_str = '' - if is_img: - result = '<img src="%s" alt="%s"%s%s' \ - % (url, link_text.replace('"', '"'), - title_str, self.empty_element_suffix) - curr_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - elif start_idx >= anchor_allowed_pos: - result = '<a href="%s"%s>%s</a>' \ - % (url, title_str, link_text) - result_head = '<a href="%s"%s>' % (url, title_str) - result = '%s%s</a>' % (result_head, link_text) - # <img> allowed from curr_pos on, <a> from - # anchor_allowed_pos on. - curr_pos = start_idx + len(result_head) - anchor_allowed_pos = start_idx + len(result) - text = text[:start_idx] + result + text[match.end():] - else: - # Anchor not allowed here. - curr_pos = start_idx + 1 - else: - # This id isn't defined, leave the markup alone. - curr_pos = match.end() - continue - - # Otherwise, it isn't markup. - curr_pos = start_idx + 1 - - return text - - - _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M) - def _setext_h_sub(self, match): - n = {"=": 1, "-": 2}[match.group(2)[0]] - demote_headers = self.extras.get("demote-headers") - if demote_headers: - n = min(n + demote_headers, 6) - return "<h%d>%s</h%d>\n\n" \ - % (n, self._run_span_gamut(match.group(1)), n) - - _atx_h_re = re.compile(r''' - ^(\#{1,6}) # \1 = string of #'s - [ \t]* - (.+?) # \2 = Header text - [ \t]* - (?<!\\) # ensure not an escaped trailing '#' - \#* # optional closing #'s (not counted) - \n+ - ''', re.X | re.M) - def _atx_h_sub(self, match): - n = len(match.group(1)) - demote_headers = self.extras.get("demote-headers") - if demote_headers: - n = min(n + demote_headers, 6) - return "<h%d>%s</h%d>\n\n" \ - % (n, self._run_span_gamut(match.group(2)), n) - - def _do_headers(self, text): - # Setext-style headers: - # Header 1 - # ======== - # - # Header 2 - # -------- - text = self._setext_h_re.sub(self._setext_h_sub, text) - - # atx-style headers: - # # Header 1 - # ## Header 2 - # ## Header 2 with closing hashes ## - # ... - # ###### Header 6 - text = self._atx_h_re.sub(self._atx_h_sub, text) - - return text - - - _marker_ul_chars = '*+-' - _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars - _marker_ul = '(?:[%s])' % _marker_ul_chars - _marker_ol = r'(?:\d+\.)' - - def _list_sub(self, match): - lst = match.group(1) - lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol" - result = self._process_list_items(lst) - if self.list_level: - return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type) - else: - return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type) - - def _do_lists(self, text): - # Form HTML ordered (numbered) and unordered (bulleted) lists. - - for marker_pat in (self._marker_ul, self._marker_ol): - # Re-usable pattern to match any entire ul or ol list: - less_than_tab = self.tab_width - 1 - whole_list = r''' - ( # \1 = whole list - ( # \2 - [ ]{0,%d} - (%s) # \3 = first list item marker - [ \t]+ - ) - (?:.+?) - ( # \4 - \Z - | - \n{2,} - (?=\S) - (?! # Negative lookahead for another list item marker - [ \t]* - %s[ \t]+ - ) - ) - ) - ''' % (less_than_tab, marker_pat, marker_pat) - - # We use a different prefix before nested lists than top-level lists. - # See extended comment in _process_list_items(). - # - # Note: There's a bit of duplication here. My original implementation - # created a scalar regex pattern as the conditional result of the test on - # $g_list_level, and then only ran the $text =~ s{...}{...}egmx - # substitution once, using the scalar as the pattern. This worked, - # everywhere except when running under MT on my hosting account at Pair - # Networks. There, this caused all rebuilds to be killed by the reaper (or - # perhaps they crashed, but that seems incredibly unlikely given that the - # same script on the same server ran fine *except* under MT. I've spent - # more time trying to figure out why this is happening than I'd like to - # admit. My only guess, backed up by the fact that this workaround works, - # is that Perl optimizes the substition when it can figure out that the - # pattern will never change, and when this optimization isn't on, we run - # afoul of the reaper. Thus, the slightly redundant code to that uses two - # static s/// patterns rather than one conditional pattern. - - if self.list_level: - sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S) - text = sub_list_re.sub(self._list_sub, text) - else: - list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list, - re.X | re.M | re.S) - text = list_re.sub(self._list_sub, text) - - return text - - _list_item_re = re.compile(r''' - (\n)? # leading line = \1 - (^[ \t]*) # leading whitespace = \2 - (%s) [ \t]+ # list marker = \3 - ((?:.+?) # list item text = \4 - (\n{1,2})) # eols = \5 - (?= \n* (\Z | \2 (%s) [ \t]+)) - ''' % (_marker_any, _marker_any), - re.M | re.X | re.S) - - _last_li_endswith_two_eols = False - def _list_item_sub(self, match): - item = match.group(4) - leading_line = match.group(1) - leading_space = match.group(2) - if leading_line or "\n\n" in item or self._last_li_endswith_two_eols: - item = self._run_block_gamut(self._outdent(item)) - else: - # Recursion for sub-lists: - item = self._do_lists(self._outdent(item)) - if item.endswith('\n'): - item = item[:-1] - item = self._run_span_gamut(item) - self._last_li_endswith_two_eols = (len(match.group(5)) == 2) - return "<li>%s</li>\n" % item - - def _process_list_items(self, list_str): - # Process the contents of a single ordered or unordered list, - # splitting it into individual list items. - - # The $g_list_level global keeps track of when we're inside a list. - # Each time we enter a list, we increment it; when we leave a list, - # we decrement. If it's zero, we're not in a list anymore. - # - # We do this because when we're not inside a list, we want to treat - # something like this: - # - # I recommend upgrading to version - # 8. Oops, now this line is treated - # as a sub-list. - # - # As a single paragraph, despite the fact that the second line starts - # with a digit-period-space sequence. - # - # Whereas when we're inside a list (or sub-list), that line will be - # treated as the start of a sub-list. What a kludge, huh? This is - # an aspect of Markdown's syntax that's hard to parse perfectly - # without resorting to mind-reading. Perhaps the solution is to - # change the syntax rules such that sub-lists must start with a - # starting cardinal number; e.g. "1." or "a.". - self.list_level += 1 - self._last_li_endswith_two_eols = False - list_str = list_str.rstrip('\n') + '\n' - list_str = self._list_item_re.sub(self._list_item_sub, list_str) - self.list_level -= 1 - return list_str - - def _get_pygments_lexer(self, lexer_name): - try: - from pygments import lexers, util - except ImportError: - return None - try: - return lexers.get_lexer_by_name(lexer_name) - except util.ClassNotFound: - return None - - def _color_with_pygments(self, codeblock, lexer, **formatter_opts): - import pygments - import pygments.formatters - - class HtmlCodeFormatter(pygments.formatters.HtmlFormatter): - def _wrap_code(self, inner): - """A function for use in a Pygments Formatter which - wraps in <code> tags. - """ - yield 0, "<code>" - for tup in inner: - yield tup - yield 0, "</code>" - - def wrap(self, source, outfile): - """Return the source with a code, pre, and div.""" - return self._wrap_div(self._wrap_pre(self._wrap_code(source))) - - formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts) - return pygments.highlight(codeblock, lexer, formatter) - - def _code_block_sub(self, match): - codeblock = match.group(1) - codeblock = self._outdent(codeblock) - codeblock = self._detab(codeblock) - codeblock = codeblock.lstrip('\n') # trim leading newlines - codeblock = codeblock.rstrip() # trim trailing whitespace - - if "code-color" in self.extras and codeblock.startswith(":::"): - lexer_name, rest = codeblock.split('\n', 1) - lexer_name = lexer_name[3:].strip() - lexer = self._get_pygments_lexer(lexer_name) - codeblock = rest.lstrip("\n") # Remove lexer declaration line. - if lexer: - formatter_opts = self.extras['code-color'] or {} - colored = self._color_with_pygments(codeblock, lexer, - **formatter_opts) - return "\n\n%s\n\n" % colored - - codeblock = self._encode_code(codeblock) - return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock - - def _do_code_blocks(self, text): - """Process Markdown `<pre><code>` blocks.""" - code_block_re = re.compile(r''' - (?:\n\n|\A) - ( # $1 = the code block -- one or more lines, starting with a space/tab - (?: - (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces - .*\n+ - )+ - ) - ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc - ''' % (self.tab_width, self.tab_width), - re.M | re.X) - - return code_block_re.sub(self._code_block_sub, text) - - - # Rules for a code span: - # - backslash escapes are not interpreted in a code span - # - to include one or or a run of more backticks the delimiters must - # be a longer run of backticks - # - cannot start or end a code span with a backtick; pad with a - # space and that space will be removed in the emitted HTML - # See `test/tm-cases/escapes.text` for a number of edge-case - # examples. - _code_span_re = re.compile(r''' - (?<!\\) - (`+) # \1 = Opening run of ` - (?!`) # See Note A test/tm-cases/escapes.text - (.+?) # \2 = The code block - (?<!`) - \1 # Matching closer - (?!`) - ''', re.X | re.S) - - def _code_span_sub(self, match): - c = match.group(2).strip(" \t") - c = self._encode_code(c) - return "<code>%s</code>" % c - - def _do_code_spans(self, text): - # * Backtick quotes are used for <code></code> spans. - # - # * You can use multiple backticks as the delimiters if you want to - # include literal backticks in the code span. So, this input: - # - # Just type ``foo `bar` baz`` at the prompt. - # - # Will translate to: - # - # <p>Just type <code>foo `bar` baz</code> at the prompt.</p> - # - # There's no arbitrary limit to the number of backticks you - # can use as delimters. If you need three consecutive backticks - # in your code, use four for delimiters, etc. - # - # * You can use spaces to get literal backticks at the edges: - # - # ... type `` `bar` `` ... - # - # Turns to: - # - # ... type <code>`bar`</code> ... - return self._code_span_re.sub(self._code_span_sub, text) - - def _encode_code(self, text): - """Encode/escape certain characters inside Markdown code runs. - The point is that in code, these characters are literals, - and lose their special Markdown meanings. - """ - replacements = [ - # Encode all ampersands; HTML entities are not - # entities within a Markdown code span. - ('&', '&'), - # Do the angle bracket song and dance: - ('<', '<'), - ('>', '>'), - # Now, escape characters that are magic in Markdown: - ('*', g_escape_table['*']), - ('_', g_escape_table['_']), - ('{', g_escape_table['{']), - ('}', g_escape_table['}']), - ('[', g_escape_table['[']), - (']', g_escape_table[']']), - ('\\', g_escape_table['\\']), - ] - for before, after in replacements: - text = text.replace(before, after) - return text - - _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S) - _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S) - _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S) - _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S) - def _do_italics_and_bold(self, text): - # <strong> must go first: - if "code-friendly" in self.extras: - text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text) - text = self._code_friendly_em_re.sub(r"<em>\1</em>", text) - else: - text = self._strong_re.sub(r"<strong>\2</strong>", text) - text = self._em_re.sub(r"<em>\2</em>", text) - return text - - - _block_quote_re = re.compile(r''' - ( # Wrap whole match in \1 - ( - ^[ \t]*>[ \t]? # '>' at the start of a line - .+\n # rest of the first line - (.+\n)* # subsequent consecutive lines - \n* # blanks - )+ - ) - ''', re.M | re.X) - _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M); - - _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S) - def _dedent_two_spaces_sub(self, match): - return re.sub(r'(?m)^ ', '', match.group(1)) - - def _block_quote_sub(self, match): - bq = match.group(1) - bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting - bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines - bq = self._run_block_gamut(bq) # recurse - - bq = re.sub('(?m)^', ' ', bq) - # These leading spaces screw with <pre> content, so we need to fix that: - bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq) - - return "<blockquote>\n%s\n</blockquote>\n\n" % bq - - def _do_block_quotes(self, text): - if '>' not in text: - return text - return self._block_quote_re.sub(self._block_quote_sub, text) - - def _form_paragraphs(self, text): - # Strip leading and trailing lines: - text = text.strip('\n') - - # Wrap <p> tags. - grafs = re.split(r"\n{2,}", text) - for i, graf in enumerate(grafs): - if graf in self.html_blocks: - # Unhashify HTML blocks - grafs[i] = self.html_blocks[graf] - else: - # Wrap <p> tags. - graf = self._run_span_gamut(graf) - grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>" - - return "\n\n".join(grafs) - - def _add_footnotes(self, text): - if self.footnotes: - footer = [ - '<div class="footnotes">', - '<hr' + self.empty_element_suffix, - '<ol>', - ] - for i, id in enumerate(self.footnote_ids): - if i != 0: - footer.append('') - footer.append('<li id="fn-%s">' % id) - footer.append(self._run_block_gamut(self.footnotes[id])) - backlink = ('<a href="#fnref-%s" ' - 'class="footnoteBackLink" ' - 'title="Jump back to footnote %d in the text.">' - '↩</a>' % (id, i+1)) - if footer[-1].endswith("</p>"): - footer[-1] = footer[-1][:-len("</p>")] \ - + ' ' + backlink + "</p>" - else: - footer.append("\n<p>%s</p>" % backlink) - footer.append('</li>') - footer.append('</ol>') - footer.append('</div>') - return text + '\n\n' + '\n'.join(footer) - else: - return text - - # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin: - # http://bumppo.net/projects/amputator/ - _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)') - _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I) - _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I) - - def _encode_amps_and_angles(self, text): - # Smart processing for ampersands and angle brackets that need - # to be encoded. - text = self._ampersand_re.sub('&', text) - - # Encode naked <'s - text = self._naked_lt_re.sub('<', text) - - # Encode naked >'s - # Note: Other markdown implementations (e.g. Markdown.pl, PHP - # Markdown) don't do this. - text = self._naked_gt_re.sub('>', text) - return text - - def _encode_backslash_escapes(self, text): - for ch, escape in g_escape_table.items(): - text = text.replace("\\"+ch, escape) - return text - - _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I) - def _auto_link_sub(self, match): - g1 = match.group(1) - return '<a href="%s">%s</a>' % (g1, g1) - - _auto_email_link_re = re.compile(r""" - < - (?:mailto:)? - ( - [-.\w]+ - \@ - [-\w]+(\.[-\w]+)*\.[a-z]+ - ) - > - """, re.I | re.X | re.U) - def _auto_email_link_sub(self, match): - return self._encode_email_address( - self._unescape_special_chars(match.group(1))) - - def _do_auto_links(self, text): - text = self._auto_link_re.sub(self._auto_link_sub, text) - text = self._auto_email_link_re.sub(self._auto_email_link_sub, text) - return text - - def _encode_email_address(self, addr): - # Input: an email address, e.g. "foo@example.com" - # - # Output: the email address as a mailto link, with each character - # of the address encoded as either a decimal or hex entity, in - # the hopes of foiling most address harvesting spam bots. E.g.: - # - # <a href="mailto:foo@e - # xample.com">foo - # @example.com</a> - # - # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk - # mailing list: <http://tinyurl.com/yu7ue> - chars = [_xml_encode_email_char_at_random(ch) - for ch in "mailto:" + addr] - # Strip the mailto: from the visible part. - addr = '<a href="%s">%s</a>' \ - % (''.join(chars), ''.join(chars[7:])) - return addr - - def _do_link_patterns(self, text): - """Caveat emptor: there isn't much guarding against link - patterns being formed inside other standard Markdown links, e.g. - inside a [link def][like this]. - - Dev Notes: *Could* consider prefixing regexes with a negative - lookbehind assertion to attempt to guard against this. - """ - link_from_hash = {} - for regex, repl in self.link_patterns: - replacements = [] - for match in regex.finditer(text): - if hasattr(repl, "__call__"): - href = repl(match) - else: - href = match.expand(repl) - replacements.append((match.span(), href)) - for (start, end), href in reversed(replacements): - escaped_href = ( - href.replace('"', '"') # b/c of attr quote - # To avoid markdown <em> and <strong>: - .replace('*', g_escape_table['*']) - .replace('_', g_escape_table['_'])) - link = '<a href="%s">%s</a>' % (escaped_href, text[start:end]) - hash = md5(link).hexdigest() - link_from_hash[hash] = link - text = text[:start] + hash + text[end:] - for hash, link in link_from_hash.items(): - text = text.replace(hash, link) - return text - - def _unescape_special_chars(self, text): - # Swap back in all the special characters we've hidden. - for ch, hash in g_escape_table.items(): - text = text.replace(hash, ch) - return text - - def _outdent(self, text): - # Remove one level of line-leading tabs or spaces - return self._outdent_re.sub('', text) - - -class MarkdownWithExtras(Markdown): - """A markdowner class that enables most extras: - - - footnotes - - code-color (only has effect if 'pygments' Python module on path) - - These are not included: - - pyshell (specific to Python-related documenting) - - code-friendly (because it *disables* part of the syntax) - - link-patterns (because you need to specify some actual - link-patterns anyway) - """ - extras = ["footnotes", "code-color"] - - -#---- internal support functions - -# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549 -def _curry(*args, **kwargs): - function, args = args[0], args[1:] - def result(*rest, **kwrest): - combined = kwargs.copy() - combined.update(kwrest) - return function(*args + rest, **combined) - return result - -# Recipe: regex_from_encoded_pattern (1.0) -def _regex_from_encoded_pattern(s): - """'foo' -> re.compile(re.escape('foo')) - '/foo/' -> re.compile('foo') - '/foo/i' -> re.compile('foo', re.I) - """ - if s.startswith('/') and s.rfind('/') != 0: - # Parse it: /PATTERN/FLAGS - idx = s.rfind('/') - pattern, flags_str = s[1:idx], s[idx+1:] - flag_from_char = { - "i": re.IGNORECASE, - "l": re.LOCALE, - "s": re.DOTALL, - "m": re.MULTILINE, - "u": re.UNICODE, - } - flags = 0 - for char in flags_str: - try: - flags |= flag_from_char[char] - except KeyError: - raise ValueError("unsupported regex flag: '%s' in '%s' " - "(must be one of '%s')" - % (char, s, ''.join(flag_from_char.keys()))) - return re.compile(s[1:idx], flags) - else: # not an encoded regex - return re.compile(re.escape(s)) - -# Recipe: dedent (0.1.2) -def _dedentlines(lines, tabsize=8, skip_first_line=False): - """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines - - "lines" is a list of lines to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - Same as dedent() except operates on a sequence of lines. Note: the - lines list is modified **in-place**. - """ - DEBUG = False - if DEBUG: - print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ - % (tabsize, skip_first_line) - indents = [] - margin = None - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - indent = 0 - for ch in line: - if ch == ' ': - indent += 1 - elif ch == '\t': - indent += tabsize - (indent % tabsize) - elif ch in '\r\n': - continue # skip all-whitespace lines - else: - break - else: - continue # skip all-whitespace lines - if DEBUG: print "dedent: indent=%d: %r" % (indent, line) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if DEBUG: print "dedent: margin=%r" % margin - - if margin is not None and margin > 0: - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - removed = 0 - for j, ch in enumerate(line): - if ch == ' ': - removed += 1 - elif ch == '\t': - removed += tabsize - (removed % tabsize) - elif ch in '\r\n': - if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line - lines[i] = lines[i][j:] - break - else: - raise ValueError("unexpected non-whitespace char %r in " - "line %r while removing %d-space margin" - % (ch, line, margin)) - if DEBUG: - print "dedent: %r: %r -> removed %d/%d"\ - % (line, ch, removed, margin) - if removed == margin: - lines[i] = lines[i][j+1:] - break - elif removed > margin: - lines[i] = ' '*(removed-margin) + lines[i][j+1:] - break - else: - if removed: - lines[i] = lines[i][removed:] - return lines - -def _dedent(text, tabsize=8, skip_first_line=False): - """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text - - "text" is the text to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - textwrap.dedent(s), but don't expand tabs to spaces - """ - lines = text.splitlines(1) - _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) - return ''.join(lines) - - -class _memoized(object): - """Decorator that caches a function's return value each time it is called. - If called later with the same arguments, the cached value is returned, and - not re-evaluated. - - http://wiki.python.org/moin/PythonDecoratorLibrary - """ - def __init__(self, func): - self.func = func - self.cache = {} - def __call__(self, *args): - try: - return self.cache[args] - except KeyError: - self.cache[args] = value = self.func(*args) - return value - except TypeError: - # uncachable -- for instance, passing a list as an argument. - # Better to not cache than to blow up entirely. - return self.func(*args) - def __repr__(self): - """Return the function's docstring.""" - return self.func.__doc__ - - -def _xml_oneliner_re_from_tab_width(tab_width): - """Standalone XML processing instruction regex.""" - return re.compile(r""" - (?: - (?<=\n\n) # Starting after a blank line - | # or - \A\n? # the beginning of the doc - ) - ( # save in $1 - [ ]{0,%d} - (?: - <\?\w+\b\s+.*?\?> # XML processing instruction - | - <\w+:\w+\b\s+.*?/> # namespaced single tag - ) - [ \t]* - (?=\n{2,}|\Z) # followed by a blank line or end of document - ) - """ % (tab_width - 1), re.X) -_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width) - -def _hr_tag_re_from_tab_width(tab_width): - return re.compile(r""" - (?: - (?<=\n\n) # Starting after a blank line - | # or - \A\n? # the beginning of the doc - ) - ( # save in \1 - [ ]{0,%d} - <(hr) # start tag = \2 - \b # word break - ([^<>])*? # - /?> # the matching end tag - [ \t]* - (?=\n{2,}|\Z) # followed by a blank line or end of document - ) - """ % (tab_width - 1), re.X) -_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width) - - -def _xml_encode_email_char_at_random(ch): - r = random() - # Roughly 10% raw, 45% hex, 45% dec. - # '@' *must* be encoded. I [John Gruber] insist. - # Issue 26: '_' must be encoded. - if r > 0.9 and ch not in "@_": - return ch - elif r < 0.45: - # The [1:] is to drop leading '0': 0x63 -> x63 - return '&#%s;' % hex(ord(ch))[1:] - else: - return '&#%s;' % ord(ch) - -def _hash_text(text): - return 'md5:'+md5(text.encode("utf-8")).hexdigest() - - -#---- mainline - -class _NoReflowFormatter(optparse.IndentedHelpFormatter): - """An optparse formatter that does NOT reflow the description.""" - def format_description(self, description): - return description or "" - -def _test(): - import doctest - doctest.testmod() - -def main(argv=None): - if argv is None: - argv = sys.argv - if not logging.root.handlers: - logging.basicConfig() - - usage = "usage: %prog [PATHS...]" - version = "%prog "+__version__ - parser = optparse.OptionParser(prog="markdown2", usage=usage, - version=version, description=cmdln_desc, - formatter=_NoReflowFormatter()) - parser.add_option("-v", "--verbose", dest="log_level", - action="store_const", const=logging.DEBUG, - help="more verbose output") - parser.add_option("--encoding", - help="specify encoding of text content") - parser.add_option("--html4tags", action="store_true", default=False, - help="use HTML 4 style for empty element tags") - parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode", - help="sanitize literal HTML: 'escape' escapes " - "HTML meta chars, 'replace' replaces with an " - "[HTML_REMOVED] note") - parser.add_option("-x", "--extras", action="append", - help="Turn on specific extra features (not part of " - "the core Markdown spec). Supported values: " - "'code-friendly' disables _/__ for emphasis; " - "'code-color' adds code-block syntax coloring; " - "'link-patterns' adds auto-linking based on patterns; " - "'footnotes' adds the footnotes syntax;" - "'xml' passes one-liner processing instructions and namespaced XML tags;" - "'pyshell' to put unindented Python interactive shell sessions in a <code> block.") - parser.add_option("--use-file-vars", - help="Look for and use Emacs-style 'markdown-extras' " - "file var to turn on extras. See " - "<http://code.google.com/p/python-markdown2/wiki/Extras>.") - parser.add_option("--link-patterns-file", - help="path to a link pattern file") - parser.add_option("--self-test", action="store_true", - help="run internal self-tests (some doctests)") - parser.add_option("--compare", action="store_true", - help="run against Markdown.pl as well (for testing)") - parser.set_defaults(log_level=logging.INFO, compare=False, - encoding="utf-8", safe_mode=None, use_file_vars=False) - opts, paths = parser.parse_args() - log.setLevel(opts.log_level) - - if opts.self_test: - return _test() - - if opts.extras: - extras = {} - for s in opts.extras: - splitter = re.compile("[,;: ]+") - for e in splitter.split(s): - if '=' in e: - ename, earg = e.split('=', 1) - try: - earg = int(earg) - except ValueError: - pass - else: - ename, earg = e, None - extras[ename] = earg - else: - extras = None - - if opts.link_patterns_file: - link_patterns = [] - f = open(opts.link_patterns_file) - try: - for i, line in enumerate(f.readlines()): - if not line.strip(): continue - if line.lstrip().startswith("#"): continue - try: - pat, href = line.rstrip().rsplit(None, 1) - except ValueError: - raise MarkdownError("%s:%d: invalid link pattern line: %r" - % (opts.link_patterns_file, i+1, line)) - link_patterns.append( - (_regex_from_encoded_pattern(pat), href)) - finally: - f.close() - else: - link_patterns = None - - from os.path import join, dirname, abspath, exists - markdown_pl = join(dirname(dirname(abspath(__file__))), "test", - "Markdown.pl") - for path in paths: - if opts.compare: - print "==== Markdown.pl ====" - perl_cmd = 'perl %s "%s"' % (markdown_pl, path) - o = os.popen(perl_cmd) - perl_html = o.read() - o.close() - sys.stdout.write(perl_html) - print "==== markdown2.py ====" - html = markdown_path(path, encoding=opts.encoding, - html4tags=opts.html4tags, - safe_mode=opts.safe_mode, - extras=extras, link_patterns=link_patterns, - use_file_vars=opts.use_file_vars) - sys.stdout.write( - html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace')) - if opts.compare: - test_dir = join(dirname(dirname(abspath(__file__))), "test") - if exists(join(test_dir, "test_markdown2.py")): - sys.path.insert(0, test_dir) - from test_markdown2 import norm_html_from_html - norm_html = norm_html_from_html(html) - norm_perl_html = norm_html_from_html(perl_html) - else: - norm_html = html - norm_perl_html = perl_html - print "==== match? %r ====" % (norm_perl_html == norm_html) - - -if __name__ == "__main__": - sys.exit( main(sys.argv) ) - diff -Nru python-tornado-2.1.0/demos/blog/templates/archive.html python-tornado-3.1.1/demos/blog/templates/archive.html --- python-tornado-2.1.0/demos/blog/templates/archive.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/archive.html 2013-08-04 19:34:21.000000000 +0000 @@ -23,7 +23,7 @@ <ul class="archive"> {% for entry in entries %} <li> - <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div> + <div class="title"><a href="/entry/{{ entry.slug }}">{{ entry.title }}</a></div> <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> </li> {% end %} diff -Nru python-tornado-2.1.0/demos/blog/templates/base.html python-tornado-3.1.1/demos/blog/templates/base.html --- python-tornado-2.1.0/demos/blog/templates/base.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/base.html 2013-08-04 19:34:21.000000000 +0000 @@ -15,7 +15,7 @@ <a href="/compose">{{ _("New post") }}</a> - <a href="/auth/logout?next={{ url_escape(request.uri) }}">{{ _("Sign out") }}</a> {% else %} - {{ _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": "/auth/login?next=" + url_escape(request.uri)} }} + {% raw _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": "/auth/login?next=" + url_escape(request.uri)} %} {% end %} </div> <h1><a href="/">{{ escape(handler.settings["blog_title"]) }}</a></h1> diff -Nru python-tornado-2.1.0/demos/blog/templates/compose.html python-tornado-3.1.1/demos/blog/templates/compose.html --- python-tornado-2.1.0/demos/blog/templates/compose.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/compose.html 2013-08-04 19:34:21.000000000 +0000 @@ -2,8 +2,8 @@ {% block body %} <form action="{{ request.path }}" method="post" class="compose"> - <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ escape(entry.title) if entry else "" }}"/></div> - <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ escape(entry.markdown) if entry else "" }}</textarea></div> + <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ entry.title if entry else "" }}"/></div> + <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ entry.markdown if entry else "" }}</textarea></div> <div> <div style="float:right"><a href="http://daringfireball.net/projects/markdown/syntax">{{ _("Syntax documentation") }}</a></div> <input type="submit" value="{{ _("Save changes") if entry else _("Publish post") }}" class="submit"/> @@ -12,7 +12,7 @@ {% if entry %} <input type="hidden" name="id" value="{{ entry.id }}"/> {% end %} - {{ xsrf_form_html() }} + {% module xsrf_form_html() %} </form> {% end %} @@ -39,4 +39,3 @@ //]]> </script> {% end %} - diff -Nru python-tornado-2.1.0/demos/blog/templates/entry.html python-tornado-3.1.1/demos/blog/templates/entry.html --- python-tornado-2.1.0/demos/blog/templates/entry.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/entry.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,5 +1,5 @@ {% extends "base.html" %} {% block body %} - {{ modules.Entry(entry) }} + {% module Entry(entry) %} {% end %} diff -Nru python-tornado-2.1.0/demos/blog/templates/feed.xml python-tornado-3.1.1/demos/blog/templates/feed.xml --- python-tornado-2.1.0/demos/blog/templates/feed.xml 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/feed.xml 2013-08-04 19:34:21.000000000 +0000 @@ -1,25 +1,25 @@ <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %} - <title>{{ escape(handler.settings["blog_title"]) }} + {{ handler.settings["blog_title"] }} {% if len(entries) > 0 %} {{ max(e.updated for e in entries).strftime(date_format) }} {% else %} {{ datetime.datetime.utcnow().strftime(date_format) }} {% end %} http://{{ request.host }}/ - - - {{ escape(handler.settings["blog_title"]) }} + + + {{ handler.settings["blog_title"] }} {% for entry in entries %} http://{{ request.host }}/entry/{{ entry.slug }} - {{ escape(entry.title) }} + {{ entry.title }} {{ entry.updated.strftime(date_format) }} {{ entry.published.strftime(date_format) }} -

{{ entry.html }}
+
{% raw entry.html %}
 {% end %} diff -Nru python-tornado-2.1.0/demos/blog/templates/home.html python-tornado-3.1.1/demos/blog/templates/home.html --- python-tornado-2.1.0/demos/blog/templates/home.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/home.html 2013-08-04 19:34:21.000000000 +0000 @@ -2,7 +2,7 @@ {% block body %} {% for entry in entries %} - {{ modules.Entry(entry) }} + {% module Entry(entry) %} {% end %} {% end %} diff -Nru python-tornado-2.1.0/demos/blog/templates/modules/entry.html python-tornado-3.1.1/demos/blog/templates/modules/entry.html --- python-tornado-2.1.0/demos/blog/templates/modules/entry.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/blog/templates/modules/entry.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,7 +1,7 @@
-

{{ escape(entry.title) }}

+

{{ entry.title }}

{{ locale.format_date(entry.published, full_format=True, shorter=True) }}
-
{{ entry.html }}
+
{% raw entry.html %}
{% if current_user %}
{{ _("Edit this post") }}
{% end %} diff -Nru python-tornado-2.1.0/demos/chat/chatdemo.py python-tornado-3.1.1/demos/chat/chatdemo.py --- python-tornado-2.1.0/demos/chat/chatdemo.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/chat/chatdemo.py 2013-08-04 19:34:21.000000000 +0000 @@ -18,86 +18,68 @@ import tornado.auth import tornado.escape import tornado.ioloop -import tornado.options import tornado.web import os.path import uuid -from tornado.options import define, options +from tornado import gen +from tornado.options import define, options, parse_command_line define("port", default=8888, help="run on the given port", type=int) -class Application(tornado.web.Application): +class MessageBuffer(object): def __init__(self): - handlers = [ - (r"/", MainHandler), - (r"/auth/login", AuthLoginHandler), - (r"/auth/logout", AuthLogoutHandler), - (r"/a/message/new", MessageNewHandler), - (r"/a/message/updates", MessageUpdatesHandler), - ] - settings = dict( - cookie_secret="43oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", - login_url="/auth/login", - template_path=os.path.join(os.path.dirname(__file__), "templates"), - static_path=os.path.join(os.path.dirname(__file__), "static"), - xsrf_cookies=True, - autoescape="xhtml_escape", - ) - tornado.web.Application.__init__(self, handlers, **settings) - - -class BaseHandler(tornado.web.RequestHandler): - def get_current_user(self): - user_json = self.get_secure_cookie("user") - if not user_json: return None - return tornado.escape.json_decode(user_json) - - -class MainHandler(BaseHandler): - @tornado.web.authenticated - def get(self): - self.render("index.html", messages=MessageMixin.cache) - - -class MessageMixin(object): - waiters = set() - cache = [] - cache_size = 200 + self.waiters = set() + self.cache = [] + self.cache_size = 200 def wait_for_messages(self, callback, cursor=None): - cls = MessageMixin if cursor: - index = 0 - for i in xrange(len(cls.cache)): - index = len(cls.cache) - i - 1 - if cls.cache[index]["id"] == cursor: break - recent = cls.cache[index + 1:] - if recent: - callback(recent) + new_count = 0 + for msg in reversed(self.cache): + if msg["id"] == cursor: + break + new_count += 1 + if new_count: + callback(self.cache[-new_count:]) return - cls.waiters.add(callback) + self.waiters.add(callback) def cancel_wait(self, callback): - cls = MessageMixin - cls.waiters.remove(callback) + self.waiters.remove(callback) def new_messages(self, messages): - cls = MessageMixin - logging.info("Sending new message to %r listeners", len(cls.waiters)) - for callback in cls.waiters: + logging.info("Sending new message to %r listeners", len(self.waiters)) + for callback in self.waiters: try: callback(messages) except: logging.error("Error in waiter callback", exc_info=True) - cls.waiters = set() - cls.cache.extend(messages) - if len(cls.cache) > self.cache_size: - cls.cache = cls.cache[-self.cache_size:] + self.waiters = set() + self.cache.extend(messages) + if len(self.cache) > self.cache_size: + self.cache = self.cache[-self.cache_size:] + +# Making this a non-singleton is left as an exercise for the reader. +global_message_buffer = MessageBuffer() -class MessageNewHandler(BaseHandler, MessageMixin): + +class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + user_json = self.get_secure_cookie("chatdemo_user") + if not user_json: return None + return tornado.escape.json_decode(user_json) + + +class MainHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + self.render("index.html", messages=global_message_buffer.cache) + + +class MessageNewHandler(BaseHandler): @tornado.web.authenticated def post(self): message = { @@ -105,21 +87,24 @@ "from": self.current_user["first_name"], "body": self.get_argument("body"), } - message["html"] = self.render_string("message.html", message=message) + # to_basestring is necessary for Python 3's json encoder, + # which doesn't accept byte strings. + message["html"] = tornado.escape.to_basestring( + self.render_string("message.html", message=message)) if self.get_argument("next", None): self.redirect(self.get_argument("next")) else: self.write(message) - self.new_messages([message]) + global_message_buffer.new_messages([message]) -class MessageUpdatesHandler(BaseHandler, MessageMixin): +class MessageUpdatesHandler(BaseHandler): @tornado.web.authenticated @tornado.web.asynchronous def post(self): cursor = self.get_argument("cursor", None) - self.wait_for_messages(self.on_new_messages, - cursor=cursor) + global_message_buffer.wait_for_messages(self.on_new_messages, + cursor=cursor) def on_new_messages(self, messages): # Closed client connection @@ -128,33 +113,44 @@ self.finish(dict(messages=messages)) def on_connection_close(self): - self.cancel_wait(self.on_new_messages) + global_message_buffer.cancel_wait(self.on_new_messages) class AuthLoginHandler(BaseHandler, tornado.auth.GoogleMixin): @tornado.web.asynchronous + @gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) + user = yield self.get_authenticated_user() + self.set_secure_cookie("chatdemo_user", + tornado.escape.json_encode(user)) + self.redirect("/") return self.authenticate_redirect(ax_attrs=["name"]) - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - self.set_secure_cookie("user", tornado.escape.json_encode(user)) - self.redirect("/") - class AuthLogoutHandler(BaseHandler): def get(self): - self.clear_cookie("user") + self.clear_cookie("chatdemo_user") self.write("You are now logged out") def main(): - tornado.options.parse_command_line() - app = Application() + parse_command_line() + app = tornado.web.Application( + [ + (r"/", MainHandler), + (r"/auth/login", AuthLoginHandler), + (r"/auth/logout", AuthLogoutHandler), + (r"/a/message/new", MessageNewHandler), + (r"/a/message/updates", MessageUpdatesHandler), + ], + cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__", + login_url="/auth/login", + template_path=os.path.join(os.path.dirname(__file__), "templates"), + static_path=os.path.join(os.path.dirname(__file__), "static"), + xsrf_cookies=True, + ) app.listen(options.port) tornado.ioloop.IOLoop.instance().start() diff -Nru python-tornado-2.1.0/demos/facebook/facebook.py python-tornado-3.1.1/demos/facebook/facebook.py --- python-tornado-2.1.0/demos/facebook/facebook.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/facebook/facebook.py 2013-08-04 19:34:21.000000000 +0000 @@ -39,7 +39,7 @@ (r"/auth/logout", AuthLogoutHandler), ] settings = dict( - cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__", login_url="/auth/login", template_path=os.path.join(os.path.dirname(__file__), "templates"), static_path=os.path.join(os.path.dirname(__file__), "static"), @@ -55,7 +55,7 @@ class BaseHandler(tornado.web.RequestHandler): def get_current_user(self): - user_json = self.get_secure_cookie("user") + user_json = self.get_secure_cookie("fbdemo_user") if not user_json: return None return tornado.escape.json_decode(user_json) @@ -92,17 +92,17 @@ self.authorize_redirect(redirect_uri=my_url, client_id=self.settings["facebook_api_key"], extra_params={"scope": "read_stream"}) - + def _on_auth(self, user): if not user: raise tornado.web.HTTPError(500, "Facebook auth failed") - self.set_secure_cookie("user", tornado.escape.json_encode(user)) + self.set_secure_cookie("fbdemo_user", tornado.escape.json_encode(user)) self.redirect(self.get_argument("next", "/")) class AuthLogoutHandler(BaseHandler, tornado.auth.FacebookGraphMixin): def get(self): - self.clear_cookie("user") + self.clear_cookie("fbdemo_user") self.redirect(self.get_argument("next", "/")) diff -Nru python-tornado-2.1.0/demos/s3server/s3server.py python-tornado-3.1.1/demos/s3server/s3server.py --- python-tornado-2.1.0/demos/s3server/s3server.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/s3server/s3server.py 2013-08-04 19:34:21.000000000 +0000 @@ -221,7 +221,7 @@ self.set_header("Content-Type", "application/unknown") self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp( info.st_mtime)) - object_file = open(path, "r") + object_file = open(path, "rb") try: self.finish(object_file.read()) finally: diff -Nru python-tornado-2.1.0/demos/twitter/home.html python-tornado-3.1.1/demos/twitter/home.html --- python-tornado-2.1.0/demos/twitter/home.html 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/demos/twitter/home.html 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,12 @@ + + + Tornado Twitter Demo + + +
    + {% for tweet in timeline %} +
  • {{ tweet['user']['screen_name'] }}: {{ tweet['text'] }}
    • + {% end %} +
+ + diff -Nru python-tornado-2.1.0/demos/twitter/twitterdemo.py python-tornado-3.1.1/demos/twitter/twitterdemo.py --- python-tornado-2.1.0/demos/twitter/twitterdemo.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/demos/twitter/twitterdemo.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,92 @@ +#!/usr/bin/env python +"""A simplistic Twitter viewer to demonstrate the use of TwitterMixin. + +To run this app, you must first register an application with Twitter: + 1) Go to https://dev.twitter.com/apps and create an application. + Your application must have a callback URL registered with Twitter. + It doesn't matter what it is, but it has to be there (Twitter won't + let you use localhost in a registered callback URL, but that won't stop + you from running this demo on localhost). + 2) Create a file called "secrets.cfg" and put your consumer key and + secret (which Twitter gives you when you register an app) in it: + twitter_consumer_key = 'asdf1234' + twitter_consumer_secret = 'qwer5678' + (you could also generate a random value for "cookie_secret" and put it + in the same file, although it's not necessary to run this demo) + 3) Run this program and go to http://localhost:8888 (by default) in your + browser. +""" + +import logging + +from tornado.auth import TwitterMixin +from tornado.escape import json_decode, json_encode +from tornado.ioloop import IOLoop +from tornado import gen +from tornado.options import define, options, parse_command_line, parse_config_file +from tornado.web import Application, RequestHandler, authenticated + +define('port', default=8888, help="port to listen on") +define('config_file', default='secrets.cfg', + help='filename for additional configuration') + +define('debug', default=False, group='application', + help="run in debug mode (with automatic reloading)") +# The following settings should probably be defined in secrets.cfg +define('twitter_consumer_key', type=str, group='application') +define('twitter_consumer_secret', type=str, group='application') +define('cookie_secret', type=str, group='application', + default='__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE__', + help="signing key for secure cookies") + +class BaseHandler(RequestHandler): + COOKIE_NAME = 'twitterdemo_user' + + def get_current_user(self): + user_json = self.get_secure_cookie(self.COOKIE_NAME) + if not user_json: + return None + return json_decode(user_json) + +class MainHandler(BaseHandler, TwitterMixin): + @authenticated + @gen.coroutine + def get(self): + timeline = yield self.twitter_request( + '/statuses/home_timeline', + access_token=self.current_user['access_token']) + self.render('home.html', timeline=timeline) + +class LoginHandler(BaseHandler, TwitterMixin): + @gen.coroutine + def get(self): + if self.get_argument('oauth_token', None): + user = yield self.get_authenticated_user() + self.set_secure_cookie(self.COOKIE_NAME, json_encode(user)) + self.redirect(self.get_argument('next', '/')) + else: + yield self.authorize_redirect(callback_uri=self.request.full_url()) + +class LogoutHandler(BaseHandler): + def get(self): + self.clear_cookie(self.COOKIE_NAME) + +def main(): + parse_command_line(final=False) + parse_config_file(options.config_file) + + app = Application( + [ + ('/', MainHandler), + ('/login', LoginHandler), + ('/logout', LogoutHandler), + ], + login_url='/login', + **options.group_dict('application')) + app.listen(options.port) + + logging.info('Listening on http://localhost:%d' % options.port) + IOLoop.instance().start() + +if __name__ == '__main__': + main() diff -Nru python-tornado-2.1.0/demos/websocket/chatdemo.py python-tornado-3.1.1/demos/websocket/chatdemo.py --- python-tornado-2.1.0/demos/websocket/chatdemo.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/websocket/chatdemo.py 2013-08-04 19:34:21.000000000 +0000 @@ -39,11 +39,10 @@ (r"/chatsocket", ChatSocketHandler), ] settings = dict( - cookie_secret="43oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__", template_path=os.path.join(os.path.dirname(__file__), "templates"), static_path=os.path.join(os.path.dirname(__file__), "static"), xsrf_cookies=True, - autoescape=None, ) tornado.web.Application.__init__(self, handlers, **settings) @@ -57,6 +56,10 @@ cache = [] cache_size = 200 + def allow_draft76(self): + # for iOS 5.0 Safari + return True + def open(self): ChatSocketHandler.waiters.add(self) @@ -85,7 +88,8 @@ "id": str(uuid.uuid4()), "body": parsed["body"], } - chat["html"] = self.render_string("message.html", message=chat) + chat["html"] = tornado.escape.to_basestring( + self.render_string("message.html", message=chat)) ChatSocketHandler.update_cache(chat) ChatSocketHandler.send_updates(chat) diff -Nru python-tornado-2.1.0/demos/websocket/static/chat.js python-tornado-3.1.1/demos/websocket/static/chat.js --- python-tornado-2.1.0/demos/websocket/static/chat.js 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/websocket/static/chat.js 2013-08-04 19:34:21.000000000 +0000 @@ -50,11 +50,8 @@ socket: null, start: function() { - if ("WebSocket" in window) { - updater.socket = new WebSocket("ws://localhost:8888/chatsocket"); - } else { - updater.socket = new MozWebSocket("ws://localhost:8888/chatsocket"); - } + var url = "ws://" + location.host + "/chatsocket"; + updater.socket = new WebSocket(url); updater.socket.onmessage = function(event) { updater.showMessage(JSON.parse(event.data)); } diff -Nru python-tornado-2.1.0/demos/websocket/templates/index.html python-tornado-3.1.1/demos/websocket/templates/index.html --- python-tornado-2.1.0/demos/websocket/templates/index.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/websocket/templates/index.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,7 +1,7 @@ - + Tornado Chat Demo @@ -20,7 +20,7 @@
- {{ xsrf_form_html() }} + {% module xsrf_form_html() %}
diff -Nru python-tornado-2.1.0/demos/websocket/templates/message.html python-tornado-3.1.1/demos/websocket/templates/message.html --- python-tornado-2.1.0/demos/websocket/templates/message.html 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/demos/websocket/templates/message.html 2013-08-04 19:34:21.000000000 +0000 @@ -1,2 +1 @@ -{% import tornado.escape %} -
{{ tornado.escape.linkify(message["body"]) }}
+
{% module linkify(message["body"]) %}
diff -Nru python-tornado-2.1.0/runtests.sh python-tornado-3.1.1/runtests.sh --- python-tornado-2.1.0/runtests.sh 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/runtests.sh 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,18 @@ +#!/bin/sh +# Run the Tornado test suite. +# +# Also consider using tox, which uses virtualenv to run the test suite +# under multiple versions of python. +# +# This script requires that `python` is python 2.x; to run the tests under +# python 3 tornado must be installed so that 2to3 is run. The easiest +# way to run the tests under python 3 is with tox: "tox -e py32". + +cd $(dirname $0) + +# "python -m" differs from "python tornado/test/runtests.py" in how it sets +# up the default python path. "python -m" uses the current directory, +# while "python file.py" uses the directory containing "file.py" (which is +# not what you want if file.py appears within a package you want to import +# from) +python -m tornado.test.runtests "$@" diff -Nru python-tornado-2.1.0/setup.cfg python-tornado-3.1.1/setup.cfg --- python-tornado-2.1.0/setup.cfg 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/setup.cfg 2013-09-01 18:44:12.000000000 +0000 @@ -0,0 +1,5 @@ +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff -Nru python-tornado-2.1.0/setup.py python-tornado-3.1.1/setup.py --- python-tornado-2.1.0/setup.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/setup.py 2013-09-01 18:41:35.000000000 +0000 @@ -25,19 +25,10 @@ kwargs = {} -# Build the epoll extension for Linux systems with Python < 2.6 -extensions = [] -major, minor = sys.version_info[:2] -python_26 = (major > 2 or (major == 2 and minor >= 6)) -if "linux" in sys.platform.lower() and not python_26: - extensions.append(distutils.core.Extension( - "tornado.epoll", ["tornado/epoll.c"])) +version = "3.1.1" -version = "2.1" - -if major >= 3: - import setuptools # setuptools is required for use_2to3 - kwargs["use_2to3"] = True +with open('README.rst') as f: + long_description = f.read() distutils.core.setup( name="tornado", @@ -45,14 +36,38 @@ packages = ["tornado", "tornado.test", "tornado.platform"], package_data = { "tornado": ["ca-certificates.crt"], - "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"], + # data files need to be listed both here (which determines what gets + # installed) and in MANIFEST.in (which determines what gets included + # in the sdist tarball) + "tornado.test": [ + "README", + "csv_translations/fr_FR.csv", + "gettext_translations/fr_FR/LC_MESSAGES/tornado_test.mo", + "gettext_translations/fr_FR/LC_MESSAGES/tornado_test.po", + "options_test.cfg", + "static/robots.txt", + "static/dir/index.html", + "templates/utf8.html", + "test.crt", + "test.key", + ], }, - ext_modules = extensions, author="Facebook", author_email="python-tornado@googlegroups.com", url="http://www.tornadoweb.org/", - download_url="http://github.com/downloads/facebook/tornado/tornado-%s.tar.gz" % version, license="http://www.apache.org/licenses/LICENSE-2.0", - description="Tornado is an open source version of the scalable, non-blocking web server and and tools that power FriendFeed", + description="Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed.", + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + ], + long_description=long_description, **kwargs ) diff -Nru python-tornado-2.1.0/tornado/__init__.py python-tornado-3.1.1/tornado/__init__.py --- python-tornado-2.1.0/tornado/__init__.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/__init__.py 2013-09-01 18:41:35.000000000 +0000 @@ -16,5 +16,14 @@ """The Tornado web server and tools.""" -version = "2.1" -version_info = (2, 1, 0) +from __future__ import absolute_import, division, print_function, with_statement + +# version is a human-readable version number. + +# version_info is a four-tuple for programmatic comparison. The first +# three numbers are the components of the version number. The fourth +# is zero for an official release, positive for a development branch, +# or negative for a release candidate or beta (after the base version +# number has been incremented) +version = "3.1.1" +version_info = (3, 1, 1, 0) diff -Nru python-tornado-2.1.0/tornado/auth.py python-tornado-3.1.1/tornado/auth.py --- python-tornado-2.1.0/tornado/auth.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/auth.py 2013-09-01 18:41:35.000000000 +0000 @@ -14,15 +14,19 @@ # License for the specific language governing permissions and limitations # under the License. -"""Implementations of various third-party authentication schemes. +"""This module contains implementations of various third-party +authentication schemes. -All the classes in this file are class Mixins designed to be used with -web.py RequestHandler classes. The primary methods for each service are -authenticate_redirect(), authorize_redirect(), and get_authenticated_user(). -The former should be called to redirect the user to, e.g., the OpenID -authentication page on the third party service, and the latter should -be called upon return to get the user data from the data returned by -the third party service. +All the classes in this file are class mixins designed to be used with +the `tornado.web.RequestHandler` class. They are used in two ways: + +* On a login handler, use methods such as ``authenticate_redirect()``, + ``authorize_redirect()``, and ``get_authenticated_user()`` to + establish the user's identity and store authentication tokens to your + database and/or cookies. +* In non-login handlers, use methods such as ``facebook_request()`` + or ``twitter_request()`` to use the authentication tokens to make + requests to the respective services. They all take slightly different arguments due to the fact all these services implement authentication and authorization slightly differently. @@ -30,80 +34,146 @@ Example usage for Google OpenID:: - class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + class GoogleLoginHandler(tornado.web.RequestHandler, + tornado.auth.GoogleMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authenticate_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - # Save the user with, e.g., set_secure_cookie() + user = yield self.get_authenticated_user() + # Save the user with e.g. set_secure_cookie() + else: + yield self.authenticate_redirect() """ +from __future__ import absolute_import, division, print_function, with_statement + import base64 import binascii +import functools import hashlib import hmac -import logging import time -import urllib -import urlparse import uuid +from tornado.concurrent import Future, chain_future, return_future +from tornado import gen from tornado import httpclient from tornado import escape from tornado.httputil import url_concat -from tornado.util import bytes_type, b +from tornado.log import gen_log +from tornado.util import bytes_type, u, unicode_type, ArgReplacer + +try: + import urlparse # py2 +except ImportError: + import urllib.parse as urlparse # py3 + +try: + import urllib.parse as urllib_parse # py3 +except ImportError: + import urllib as urllib_parse # py2 + + +class AuthError(Exception): + pass + + +def _auth_future_to_callback(callback, future): + try: + result = future.result() + except AuthError as e: + gen_log.warning(str(e)) + result = None + callback(result) + + +def _auth_return_future(f): + """Similar to tornado.concurrent.return_future, but uses the auth + module's legacy callback interface. + + Note that when using this decorator the ``callback`` parameter + inside the function will actually be a future. + """ + replacer = ArgReplacer(f, 'callback') + + @functools.wraps(f) + def wrapper(*args, **kwargs): + future = Future() + callback, args, kwargs = replacer.replace(future, args, kwargs) + if callback is not None: + future.add_done_callback( + functools.partial(_auth_future_to_callback, callback)) + f(*args, **kwargs) + return future + return wrapper + class OpenIdMixin(object): """Abstract implementation of OpenID and Attribute Exchange. - See GoogleMixin below for example implementations. + See `GoogleMixin` below for a customized example (which also + includes OAuth support). + + Class attributes: + + * ``_OPENID_ENDPOINT``: the identity provider's URI. """ + @return_future def authenticate_redirect(self, callback_uri=None, - ax_attrs=["name","email","language","username"]): - """Returns the authentication URL for this service. + ax_attrs=["name", "email", "language", "username"], + callback=None): + """Redirects to the authentication URL for this service. After authentication, the service will redirect back to the given - callback URI. + callback URI with additional parameters including ``openid.mode``. We request the given attributes for the authenticated user by default (name, email, language, and username). If you don't need all those attributes for your app, you can request fewer with the ax_attrs keyword argument. + + .. versionchanged:: 3.1 + Returns a `.Future` and takes an optional callback. These are + not strictly necessary as this method is synchronous, + but they are supplied for consistency with + `OAuthMixin.authorize_redirect`. """ callback_uri = callback_uri or self.request.uri args = self._openid_args(callback_uri, ax_attrs=ax_attrs) - self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)) + self.redirect(self._OPENID_ENDPOINT + "?" + urllib_parse.urlencode(args)) + callback() + @_auth_return_future def get_authenticated_user(self, callback, http_client=None): """Fetches the authenticated user data upon redirect. This method should be called by the handler that receives the - redirect from the authenticate_redirect() or authorize_redirect() - methods. + redirect from the `authenticate_redirect()` method (which is + often the same as the one that calls it; in that case you would + call `get_authenticated_user` if the ``openid.mode`` parameter + is present and `authenticate_redirect` if it is not). + + The result of this method will generally be used to set a cookie. """ # Verify the OpenID response via direct request to the OP - args = dict((k, v[-1]) for k, v in self.request.arguments.iteritems()) - args["openid.mode"] = u"check_authentication" + args = dict((k, v[-1]) for k, v in self.request.arguments.items()) + args["openid.mode"] = u("check_authentication") url = self._OPENID_ENDPOINT - if http_client is None: http_client = httpclient.AsyncHTTPClient() + if http_client is None: + http_client = self.get_auth_http_client() http_client.fetch(url, self.async_callback( self._on_authentication_verified, callback), - method="POST", body=urllib.urlencode(args)) + method="POST", body=urllib_parse.urlencode(args)) def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): url = urlparse.urljoin(self.request.full_url(), callback_uri) args = { "openid.ns": "http://specs.openid.net/auth/2.0", "openid.claimed_id": - "http://specs.openid.net/auth/2.0/identifier_select", + "http://specs.openid.net/auth/2.0/identifier_select", "openid.identity": - "http://specs.openid.net/auth/2.0/identifier_select", + "http://specs.openid.net/auth/2.0/identifier_select", "openid.return_to": url, "openid.realm": urlparse.urljoin(url, '/'), "openid.mode": "checkid_setup", @@ -120,11 +190,11 @@ required += ["firstname", "fullname", "lastname"] args.update({ "openid.ax.type.firstname": - "http://axschema.org/namePerson/first", + "http://axschema.org/namePerson/first", "openid.ax.type.fullname": - "http://axschema.org/namePerson", + "http://axschema.org/namePerson", "openid.ax.type.lastname": - "http://axschema.org/namePerson/last", + "http://axschema.org/namePerson/last", }) known_attrs = { "email": "http://axschema.org/contact/email", @@ -138,37 +208,40 @@ if oauth_scope: args.update({ "openid.ns.oauth": - "http://specs.openid.net/extensions/oauth/1.0", + "http://specs.openid.net/extensions/oauth/1.0", "openid.oauth.consumer": self.request.host.split(":")[0], "openid.oauth.scope": oauth_scope, }) return args - def _on_authentication_verified(self, callback, response): - if response.error or b("is_valid:true") not in response.body: - logging.warning("Invalid OpenID response: %s", response.error or - response.body) - callback(None) + def _on_authentication_verified(self, future, response): + if response.error or b"is_valid:true" not in response.body: + future.set_exception(AuthError( + "Invalid OpenID response: %s" % (response.error or + response.body))) return # Make sure we got back at least an email from attribute exchange ax_ns = None - for name in self.request.arguments.iterkeys(): + for name in self.request.arguments: if name.startswith("openid.ns.") and \ - self.get_argument(name) == u"http://openid.net/srv/ax/1.0": + self.get_argument(name) == u("http://openid.net/srv/ax/1.0"): ax_ns = name[10:] break + def get_ax_arg(uri): - if not ax_ns: return u"" + if not ax_ns: + return u("") prefix = "openid." + ax_ns + ".type." ax_name = None - for name in self.request.arguments.iterkeys(): + for name in self.request.arguments.keys(): if self.get_argument(name) == uri and name.startswith(prefix): part = name[len(prefix):] ax_name = "openid." + ax_ns + ".value." + part break - if not ax_name: return u"" - return self.get_argument(ax_name, u"") + if not ax_name: + return u("") + return self.get_argument(ax_name, u("")) email = get_ax_arg("http://axschema.org/contact/email") name = get_ax_arg("http://axschema.org/namePerson") @@ -187,39 +260,76 @@ if name: user["name"] = name elif name_parts: - user["name"] = u" ".join(name_parts) + user["name"] = u(" ").join(name_parts) elif email: user["name"] = email.split("@")[0] - if email: user["email"] = email - if locale: user["locale"] = locale - if username: user["username"] = username - callback(user) + if email: + user["email"] = email + if locale: + user["locale"] = locale + if username: + user["username"] = username + claimed_id = self.get_argument("openid.claimed_id", None) + if claimed_id: + user["claimed_id"] = claimed_id + future.set_result(user) + + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() class OAuthMixin(object): - """Abstract implementation of OAuth. + """Abstract implementation of OAuth 1.0 and 1.0a. - See TwitterMixin and FriendFeedMixin below for example implementations. - """ + See `TwitterMixin` and `FriendFeedMixin` below for example implementations, + or `GoogleMixin` for an OAuth/OpenID hybrid. + + Class attributes: + * ``_OAUTH_AUTHORIZE_URL``: The service's OAuth authorization url. + * ``_OAUTH_ACCESS_TOKEN_URL``: The service's OAuth access token url. + * ``_OAUTH_VERSION``: May be either "1.0" or "1.0a". + * ``_OAUTH_NO_CALLBACKS``: Set this to True if the service requires + advance registration of callbacks. + + Subclasses must also override the `_oauth_get_user_future` and + `_oauth_consumer_token` methods. + """ + @return_future def authorize_redirect(self, callback_uri=None, extra_params=None, - http_client=None): + http_client=None, callback=None): """Redirects the user to obtain OAuth authorization for this service. - Twitter and FriendFeed both require that you register a Callback - URL with your application. You should call this method to log the - user in, and then call get_authenticated_user() in the handler - you registered as your Callback URL to complete the authorization - process. + The ``callback_uri`` may be omitted if you have previously + registered a callback URI with the third-party service. For + some sevices (including Friendfeed), you must use a + previously-registered callback URI and cannot specify a + callback via this method. - This method sets a cookie called _oauth_request_token which is - subsequently used (and cleared) in get_authenticated_user for + This method sets a cookie called ``_oauth_request_token`` which is + subsequently used (and cleared) in `get_authenticated_user` for security purposes. + + Note that this method is asynchronous, although it calls + `.RequestHandler.finish` for you so it may not be necessary + to pass a callback or use the `.Future` it returns. However, + if this method is called from a function decorated with + `.gen.coroutine`, you must call it with ``yield`` to keep the + response from being closed prematurely. + + .. versionchanged:: 3.1 + Now returns a `.Future` and takes an optional callback, for + compatibility with `.gen.coroutine`. """ if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): raise Exception("This service does not support oauth_callback") if http_client is None: - http_client = httpclient.AsyncHTTPClient() + http_client = self.get_auth_http_client() if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": http_client.fetch( self._oauth_request_token_url(callback_uri=callback_uri, @@ -227,96 +337,107 @@ self.async_callback( self._on_request_token, self._OAUTH_AUTHORIZE_URL, - callback_uri)) + callback_uri, + callback)) else: http_client.fetch( self._oauth_request_token_url(), self.async_callback( self._on_request_token, self._OAUTH_AUTHORIZE_URL, - callback_uri)) - + callback_uri, + callback)) + @_auth_return_future def get_authenticated_user(self, callback, http_client=None): - """Gets the OAuth authorized user and access token on callback. - - This method should be called from the handler for your registered - OAuth Callback URL to complete the registration process. We call - callback with the authenticated user, which in addition to standard - attributes like 'name' includes the 'access_key' attribute, which - contains the OAuth access you can use to make authorized requests - to this service on behalf of the user. + """Gets the OAuth authorized user and access token. + This method should be called from the handler for your + OAuth callback URL to complete the registration process. We run the + callback with the authenticated user dictionary. This dictionary + will contain an ``access_key`` which can be used to make authorized + requests to this service on behalf of the user. The dictionary will + also contain other fields such as ``name``, depending on the service + used. """ + future = callback request_key = escape.utf8(self.get_argument("oauth_token")) oauth_verifier = self.get_argument("oauth_verifier", None) request_cookie = self.get_cookie("_oauth_request_token") if not request_cookie: - logging.warning("Missing OAuth request token cookie") - callback(None) + future.set_exception(AuthError( + "Missing OAuth request token cookie")) return self.clear_cookie("_oauth_request_token") cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] if cookie_key != request_key: - logging.info((cookie_key, request_key, request_cookie)) - logging.warning("Request token does not match cookie") - callback(None) + future.set_exception(AuthError( + "Request token does not match cookie")) return token = dict(key=cookie_key, secret=cookie_secret) if oauth_verifier: token["verifier"] = oauth_verifier if http_client is None: - http_client = httpclient.AsyncHTTPClient() + http_client = self.get_auth_http_client() http_client.fetch(self._oauth_access_token_url(token), self.async_callback(self._on_access_token, callback)) - def _oauth_request_token_url(self, callback_uri= None, extra_params=None): + def _oauth_request_token_url(self, callback_uri=None, extra_params=None): consumer_token = self._oauth_consumer_token() url = self._OAUTH_REQUEST_TOKEN_URL args = dict( - oauth_consumer_key=consumer_token["key"], + oauth_consumer_key=escape.to_basestring(consumer_token["key"]), oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), - oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), + oauth_version="1.0", ) if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - if callback_uri: + if callback_uri == "oob": + args["oauth_callback"] = "oob" + elif callback_uri: args["oauth_callback"] = urlparse.urljoin( self.request.full_url(), callback_uri) - if extra_params: args.update(extra_params) + if extra_params: + args.update(extra_params) signature = _oauth10a_signature(consumer_token, "GET", url, args) else: signature = _oauth_signature(consumer_token, "GET", url, args) args["oauth_signature"] = signature - return url + "?" + urllib.urlencode(args) + return url + "?" + urllib_parse.urlencode(args) - def _on_request_token(self, authorize_url, callback_uri, response): + def _on_request_token(self, authorize_url, callback_uri, callback, + response): if response.error: - raise Exception("Could not get request token") + raise Exception("Could not get request token: %s" % response.error) request_token = _oauth_parse_response(response.body) - data = (base64.b64encode(request_token["key"]) + b("|") + - base64.b64encode(request_token["secret"])) + data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + + base64.b64encode(escape.utf8(request_token["secret"]))) self.set_cookie("_oauth_request_token", data) args = dict(oauth_token=request_token["key"]) - if callback_uri: + if callback_uri == "oob": + self.finish(authorize_url + "?" + urllib_parse.urlencode(args)) + callback() + return + elif callback_uri: args["oauth_callback"] = urlparse.urljoin( self.request.full_url(), callback_uri) - self.redirect(authorize_url + "?" + urllib.urlencode(args)) + self.redirect(authorize_url + "?" + urllib_parse.urlencode(args)) + callback() def _oauth_access_token_url(self, request_token): consumer_token = self._oauth_consumer_token() url = self._OAUTH_ACCESS_TOKEN_URL args = dict( - oauth_consumer_key=consumer_token["key"], - oauth_token=request_token["key"], + oauth_consumer_key=escape.to_basestring(consumer_token["key"]), + oauth_token=escape.to_basestring(request_token["key"]), oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), - oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), + oauth_version="1.0", ) if "verifier" in request_token: - args["oauth_verifier"]=request_token["verifier"] + args["oauth_verifier"] = request_token["verifier"] if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": signature = _oauth10a_signature(consumer_token, "GET", url, args, @@ -326,27 +447,57 @@ request_token) args["oauth_signature"] = signature - return url + "?" + urllib.urlencode(args) + return url + "?" + urllib_parse.urlencode(args) - def _on_access_token(self, callback, response): + def _on_access_token(self, future, response): if response.error: - logging.warning("Could not fetch access token") - callback(None) + future.set_exception(AuthError("Could not fetch access token")) return access_token = _oauth_parse_response(response.body) - self._oauth_get_user(access_token, self.async_callback( - self._on_oauth_get_user, access_token, callback)) + self._oauth_get_user_future(access_token).add_done_callback( + self.async_callback(self._on_oauth_get_user, access_token, future)) + + def _oauth_consumer_token(self): + """Subclasses must override this to return their OAuth consumer keys. + + The return value should be a `dict` with keys ``key`` and ``secret``. + """ + raise NotImplementedError() + + @return_future + def _oauth_get_user_future(self, access_token, callback): + """Subclasses must override this to get basic information about the + user. + + Should return a `.Future` whose result is a dictionary + containing information about the user, which may have been + retrieved by using ``access_token`` to make a request to the + service. + + The access token will be added to the returned dictionary to make + the result of `get_authenticated_user`. + + For backwards compatibility, the callback-based ``_oauth_get_user`` + method is also supported. + """ + # By default, call the old-style _oauth_get_user, but new code + # should override this method instead. + self._oauth_get_user(access_token, callback) def _oauth_get_user(self, access_token, callback): raise NotImplementedError() - def _on_oauth_get_user(self, access_token, callback, user): + def _on_oauth_get_user(self, access_token, future, user_future): + if user_future.exception() is not None: + future.set_exception(user_future.exception()) + return + user = user_future.result() if not user: - callback(None) + future.set_exception(AuthError("Error getting user")) return user["access_token"] = access_token - callback(user) + future.set_result(user) def _oauth_request_parameters(self, url, access_token, parameters={}, method="GET"): @@ -357,47 +508,73 @@ """ consumer_token = self._oauth_consumer_token() base_args = dict( - oauth_consumer_key=consumer_token["key"], - oauth_token=access_token["key"], + oauth_consumer_key=escape.to_basestring(consumer_token["key"]), + oauth_token=escape.to_basestring(access_token["key"]), oauth_signature_method="HMAC-SHA1", oauth_timestamp=str(int(time.time())), - oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), - oauth_version=getattr(self, "_OAUTH_VERSION", "1.0a"), + oauth_nonce=escape.to_basestring(binascii.b2a_hex(uuid.uuid4().bytes)), + oauth_version="1.0", ) args = {} args.update(base_args) args.update(parameters) if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": signature = _oauth10a_signature(consumer_token, method, url, args, - access_token) + access_token) else: signature = _oauth_signature(consumer_token, method, url, args, access_token) - base_args["oauth_signature"] = signature + base_args["oauth_signature"] = escape.to_basestring(signature) return base_args + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + + class OAuth2Mixin(object): - """Abstract implementation of OAuth v 2.""" + """Abstract implementation of OAuth 2.0. + See `FacebookGraphMixin` below for an example implementation. + + Class attributes: + + * ``_OAUTH_AUTHORIZE_URL``: The service's authorization url. + * ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url. + """ + @return_future def authorize_redirect(self, redirect_uri=None, client_id=None, - client_secret=None, extra_params=None ): + client_secret=None, extra_params=None, + callback=None): """Redirects the user to obtain OAuth authorization for this service. - Some providers require that you register a Callback - URL with your application. You should call this method to log the - user in, and then call get_authenticated_user() in the handler - you registered as your Callback URL to complete the authorization - process. + Some providers require that you register a redirect URL with + your application instead of passing one via this method. You + should call this method to log the user in, and then call + ``get_authenticated_user`` in the handler for your + redirect URL to complete the authorization process. + + .. versionchanged:: 3.1 + Returns a `.Future` and takes an optional callback. These are + not strictly necessary as this method is synchronous, + but they are supplied for consistency with + `OAuthMixin.authorize_redirect`. """ args = { - "redirect_uri": redirect_uri, - "client_id": client_id + "redirect_uri": redirect_uri, + "client_id": client_id } - if extra_params: args.update(extra_params) + if extra_params: + args.update(extra_params) self.redirect( - url_concat(self._OAUTH_AUTHORIZE_URL, args)) + url_concat(self._OAUTH_AUTHORIZE_URL, args)) + callback() - def _oauth_request_token_url(self, redirect_uri= None, client_id = None, + def _oauth_request_token_url(self, redirect_uri=None, client_id=None, client_secret=None, code=None, extra_params=None): url = self._OAUTH_ACCESS_TOKEN_URL @@ -406,101 +583,110 @@ code=code, client_id=client_id, client_secret=client_secret, - ) - if extra_params: args.update(extra_params) + ) + if extra_params: + args.update(extra_params) return url_concat(url, args) + class TwitterMixin(OAuthMixin): """Twitter OAuth authentication. To authenticate with Twitter, register your application with - Twitter at http://twitter.com/apps. Then copy your Consumer Key and - Consumer Secret to the application settings 'twitter_consumer_key' and - 'twitter_consumer_secret'. Use this Mixin on the handler for the URL - you registered as your application's Callback URL. + Twitter at http://twitter.com/apps. Then copy your Consumer Key + and Consumer Secret to the application + `~tornado.web.Application.settings` ``twitter_consumer_key`` and + ``twitter_consumer_secret``. Use this mixin on the handler for the + URL you registered as your application's callback URL. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with Twitter and get access to their stream:: - class TwitterHandler(tornado.web.RequestHandler, - tornado.auth.TwitterMixin): + class TwitterLoginHandler(tornado.web.RequestHandler, + tornado.auth.TwitterMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authorize_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Twitter auth failed") - # Save the user using, e.g., set_secure_cookie() - - The user object returned by get_authenticated_user() includes the - attributes 'username', 'name', and all of the custom Twitter user - attributes describe at - http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show - in addition to 'access_token'. You should save the access token with - the user; it is required to make requests on behalf of the user later - with twitter_request(). + user = yield self.get_authenticated_user() + # Save the user using e.g. set_secure_cookie() + else: + yield self.authorize_redirect() + + The user object returned by `~OAuthMixin.get_authenticated_user` + includes the attributes ``username``, ``name``, ``access_token``, + and all of the custom Twitter user attributes described at + https://dev.twitter.com/docs/api/1.1/get/users/show """ - _OAUTH_REQUEST_TOKEN_URL = "http://api.twitter.com/oauth/request_token" - _OAUTH_ACCESS_TOKEN_URL = "http://api.twitter.com/oauth/access_token" - _OAUTH_AUTHORIZE_URL = "http://api.twitter.com/oauth/authorize" - _OAUTH_AUTHENTICATE_URL = "http://api.twitter.com/oauth/authenticate" + _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" + _OAUTH_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" + _OAUTH_AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize" + _OAUTH_AUTHENTICATE_URL = "https://api.twitter.com/oauth/authenticate" _OAUTH_NO_CALLBACKS = False + _TWITTER_BASE_URL = "https://api.twitter.com/1.1" - - def authenticate_redirect(self): - """Just like authorize_redirect(), but auto-redirects if authorized. + @return_future + def authenticate_redirect(self, callback_uri=None, callback=None): + """Just like `~OAuthMixin.authorize_redirect`, but + auto-redirects if authorized. This is generally the right interface to use if you are using Twitter for single-sign on. - """ - http = httpclient.AsyncHTTPClient() - http.fetch(self._oauth_request_token_url(), self.async_callback( - self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None)) - def twitter_request(self, path, callback, access_token=None, - post_args=None, **args): - """Fetches the given API path, e.g., "/statuses/user_timeline/btaylor" + .. versionchanged:: 3.1 + Now returns a `.Future` and takes an optional callback, for + compatibility with `.gen.coroutine`. + """ + http = self.get_auth_http_client() + http.fetch(self._oauth_request_token_url(callback_uri=callback_uri), + self.async_callback( + self._on_request_token, self._OAUTH_AUTHENTICATE_URL, + None, callback)) + + @_auth_return_future + def twitter_request(self, path, callback=None, access_token=None, + post_args=None, **args): + """Fetches the given API path, e.g., ``statuses/user_timeline/btaylor`` - The path should not include the format (we automatically append - ".json" and parse the JSON output). + The path should not include the format or API version number. + (we automatically use JSON format and API version 1). - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. - All the Twitter methods are documented at - http://apiwiki.twitter.com/Twitter-API-Documentation. + All the Twitter methods are documented at http://dev.twitter.com/ - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuthMixin.authorize_redirect` and + `~OAuthMixin.get_authenticated_user`. The user returned through that + process includes an 'access_token' attribute that can be used + to make authenticated requests via this method. Example + usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.TwitterMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.twitter_request( + new_entry = yield self.twitter_request( "/statuses/update", post_args={"status": "Testing Tornado Web Server"}, - access_token=user["access_token"], - callback=self.async_callback(self._on_post)) - - def _on_post(self, new_entry): + access_token=self.current_user["access_token"]) if not new_entry: # Call failed; perhaps missing permission? - self.authorize_redirect() + yield self.authorize_redirect() return self.finish("Posted a message!") """ + if path.startswith('http:') or path.startswith('https:'): + # Raw urls are useful for e.g. search which doesn't follow the + # usual pattern: http://search.twitter.com/search.json + url = path + else: + url = self._TWITTER_BASE_URL + path + ".json" # Add the OAuth resource request signature if we have credentials - url = "http://api.twitter.com/1" + path + ".json" if access_token: all_args = {} all_args.update(args) @@ -509,22 +695,23 @@ oauth = self._oauth_request_parameters( url, access_token, all_args, method=method) args.update(oauth) - if args: url += "?" + urllib.urlencode(args) - callback = self.async_callback(self._on_twitter_request, callback) - http = httpclient.AsyncHTTPClient() + if args: + url += "?" + urllib_parse.urlencode(args) + http = self.get_auth_http_client() + http_callback = self.async_callback(self._on_twitter_request, callback) if post_args is not None: - http.fetch(url, method="POST", body=urllib.urlencode(post_args), - callback=callback) + http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), + callback=http_callback) else: - http.fetch(url, callback=callback) + http.fetch(url, callback=http_callback) - def _on_twitter_request(self, callback, response): + def _on_twitter_request(self, future, response): if response.error: - logging.warning("Error response %s fetching %s", response.error, - response.request.url) - callback(None) + future.set_exception(AuthError( + "Error response %s fetching %s" % (response.error, + response.request.url))) return - callback(escape.json_decode(response.body)) + future.set_result(escape.json_decode(response.body)) def _oauth_consumer_token(self): self.require_setting("twitter_consumer_key", "Twitter OAuth") @@ -533,50 +720,45 @@ key=self.settings["twitter_consumer_key"], secret=self.settings["twitter_consumer_secret"]) - def _oauth_get_user(self, access_token, callback): - callback = self.async_callback(self._parse_user_response, callback) - self.twitter_request( - "/users/show/" + access_token["screen_name"], - access_token=access_token, callback=callback) - - def _parse_user_response(self, callback, user): + @gen.coroutine + def _oauth_get_user_future(self, access_token): + user = yield self.twitter_request( + "/account/verify_credentials", + access_token=access_token) if user: user["username"] = user["screen_name"] - callback(user) + raise gen.Return(user) class FriendFeedMixin(OAuthMixin): """FriendFeed OAuth authentication. To authenticate with FriendFeed, register your application with - FriendFeed at http://friendfeed.com/api/applications. Then - copy your Consumer Key and Consumer Secret to the application settings - 'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use - this Mixin on the handler for the URL you registered as your - application's Callback URL. + FriendFeed at http://friendfeed.com/api/applications. Then copy + your Consumer Key and Consumer Secret to the application + `~tornado.web.Application.settings` ``friendfeed_consumer_key`` + and ``friendfeed_consumer_secret``. Use this mixin on the handler + for the URL you registered as your application's Callback URL. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with FriendFeed and get access to their feed:: - class FriendFeedHandler(tornado.web.RequestHandler, - tornado.auth.FriendFeedMixin): + class FriendFeedLoginHandler(tornado.web.RequestHandler, + tornado.auth.FriendFeedMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("oauth_token", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authorize_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "FriendFeed auth failed") - # Save the user using, e.g., set_secure_cookie() - - The user object returned by get_authenticated_user() includes the - attributes 'username', 'name', and 'description' in addition to - 'access_token'. You should save the access token with the user; + user = yield self.get_authenticated_user() + # Save the user using e.g. set_secure_cookie() + else: + yield self.authorize_redirect() + + The user object returned by `~OAuthMixin.get_authenticated_user()` includes the + attributes ``username``, ``name``, and ``description`` in addition to + ``access_token``. You should save the access token with the user; it is required to make requests on behalf of the user later with - friendfeed_request(). + `friendfeed_request()`. """ _OAUTH_VERSION = "1.0" _OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token" @@ -585,38 +767,40 @@ _OAUTH_NO_CALLBACKS = True _OAUTH_VERSION = "1.0" - + @_auth_return_future def friendfeed_request(self, path, callback, access_token=None, post_args=None, **args): """Fetches the given relative API path, e.g., "/bret/friends" - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. All the FriendFeed methods are documented at http://friendfeed.com/api/documentation. - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuthMixin.authorize_redirect` and + `~OAuthMixin.get_authenticated_user`. The user returned + through that process includes an ``access_token`` attribute that + can be used to make authenticated requests via this + method. + + Example usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.FriendFeedMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.friendfeed_request( + new_entry = yield self.friendfeed_request( "/entry", post_args={"body": "Testing Tornado Web Server"}, - access_token=self.current_user["access_token"], - callback=self.async_callback(self._on_post)) + access_token=self.current_user["access_token"]) - def _on_post(self, new_entry): if not new_entry: # Call failed; perhaps missing permission? - self.authorize_redirect() + yield self.authorize_redirect() return self.finish("Posted a message!") @@ -631,22 +815,23 @@ oauth = self._oauth_request_parameters( url, access_token, all_args, method=method) args.update(oauth) - if args: url += "?" + urllib.urlencode(args) + if args: + url += "?" + urllib_parse.urlencode(args) callback = self.async_callback(self._on_friendfeed_request, callback) - http = httpclient.AsyncHTTPClient() + http = self.get_auth_http_client() if post_args is not None: - http.fetch(url, method="POST", body=urllib.urlencode(post_args), + http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), callback=callback) else: http.fetch(url, callback=callback) - def _on_friendfeed_request(self, callback, response): + def _on_friendfeed_request(self, future, response): if response.error: - logging.warning("Error response %s fetching %s", response.error, - response.request.url) - callback(None) + future.set_exception(AuthError( + "Error response %s fetching %s" % (response.error, + response.request.url))) return - callback(escape.json_decode(response.body)) + future.set_result(escape.json_decode(response.body)) def _oauth_consumer_token(self): self.require_setting("friendfeed_consumer_key", "FriendFeed OAuth") @@ -655,12 +840,14 @@ key=self.settings["friendfeed_consumer_key"], secret=self.settings["friendfeed_consumer_secret"]) - def _oauth_get_user(self, access_token, callback): - callback = self.async_callback(self._parse_user_response, callback) - self.friendfeed_request( + @gen.coroutine + def _oauth_get_user_future(self, access_token, callback): + user = yield self.friendfeed_request( "/feedinfo/" + access_token["username"], - include="id,name,description", access_token=access_token, - callback=callback) + include="id,name,description", access_token=access_token) + if user: + user["username"] = user["id"] + callback(user) def _parse_user_response(self, callback, user): if user: @@ -671,35 +858,42 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): """Google Open ID / OAuth authentication. - No application registration is necessary to use Google for authentication - or to access Google resources on behalf of a user. To authenticate with - Google, redirect with authenticate_redirect(). On return, parse the - response with get_authenticated_user(). We send a dict containing the - values for the user, including 'email', 'name', and 'locale'. + No application registration is necessary to use Google for + authentication or to access Google resources on behalf of a user. + + Google implements both OpenID and OAuth in a hybrid mode. If you + just need the user's identity, use + `~OpenIdMixin.authenticate_redirect`. If you need to make + requests to Google on behalf of the user, use + `authorize_redirect`. On return, parse the response with + `~OpenIdMixin.get_authenticated_user`. We send a dict containing + the values for the user, including ``email``, ``name``, and + ``locale``. + Example usage:: - class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + class GoogleLoginHandler(tornado.web.RequestHandler, + tornado.auth.GoogleMixin): @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): if self.get_argument("openid.mode", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) - return - self.authenticate_redirect() - - def _on_auth(self, user): - if not user: - raise tornado.web.HTTPError(500, "Google auth failed") - # Save the user with, e.g., set_secure_cookie() - + user = yield self.get_authenticated_user() + # Save the user with e.g. set_secure_cookie() + else: + yield self.authenticate_redirect() """ _OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud" _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken" + @return_future def authorize_redirect(self, oauth_scope, callback_uri=None, - ax_attrs=["name","email","language","username"]): + ax_attrs=["name", "email", "language", "username"], + callback=None): """Authenticates and authorizes for the given Google resource. - Some of the available resources are: + Some of the available resources which can be used in the ``oauth_scope`` + argument are: * Gmail Contacts - http://www.google.com/m8/feeds/ * Calendar - http://www.google.com/calendar/feeds/ @@ -707,29 +901,38 @@ You can authorize multiple resources by separating the resource URLs with a space. + + .. versionchanged:: 3.1 + Returns a `.Future` and takes an optional callback. These are + not strictly necessary as this method is synchronous, + but they are supplied for consistency with + `OAuthMixin.authorize_redirect`. """ callback_uri = callback_uri or self.request.uri args = self._openid_args(callback_uri, ax_attrs=ax_attrs, oauth_scope=oauth_scope) - self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)) + self.redirect(self._OPENID_ENDPOINT + "?" + urllib_parse.urlencode(args)) + callback() + @_auth_return_future def get_authenticated_user(self, callback): """Fetches the authenticated user data upon redirect.""" # Look to see if we are doing combined OpenID/OAuth oauth_ns = "" - for name, values in self.request.arguments.iteritems(): + for name, values in self.request.arguments.items(): if name.startswith("openid.ns.") and \ - values[-1] == u"http://specs.openid.net/extensions/oauth/1.0": + values[-1] == b"http://specs.openid.net/extensions/oauth/1.0": oauth_ns = name[10:] break token = self.get_argument("openid." + oauth_ns + ".request_token", "") if token: - http = httpclient.AsyncHTTPClient() + http = self.get_auth_http_client() token = dict(key=token, secret="") http.fetch(self._oauth_access_token_url(token), self.async_callback(self._on_access_token, callback)) else: - OpenIdMixin.get_authenticated_user(self, callback) + chain_future(OpenIdMixin.get_authenticated_user(self), + callback) def _oauth_consumer_token(self): self.require_setting("google_consumer_key", "Google OAuth") @@ -738,21 +941,23 @@ key=self.settings["google_consumer_key"], secret=self.settings["google_consumer_secret"]) - def _oauth_get_user(self, access_token, callback): - OpenIdMixin.get_authenticated_user(self, callback) + def _oauth_get_user_future(self, access_token): + return OpenIdMixin.get_authenticated_user(self) + class FacebookMixin(object): """Facebook Connect authentication. - New applications should consider using `FacebookGraphMixin` below instead - of this class. + *Deprecated:* New applications should use `FacebookGraphMixin` + below instead of this class. This class does not support the + Future-based interface seen on other classes in this module. To authenticate with Facebook, register your application with Facebook at http://www.facebook.com/developers/apps.php. Then copy your API Key and Application Secret to the application settings - 'facebook_api_key' and 'facebook_secret'. + ``facebook_api_key`` and ``facebook_secret``. - When your application is set up, you can use this Mixin like this + When your application is set up, you can use this mixin like this to authenticate the user with Facebook:: class FacebookHandler(tornado.web.RequestHandler, @@ -762,22 +967,30 @@ if self.get_argument("session", None): self.get_authenticated_user(self.async_callback(self._on_auth)) return - self.authenticate_redirect() + yield self.authenticate_redirect() def _on_auth(self, user): if not user: raise tornado.web.HTTPError(500, "Facebook auth failed") # Save the user using, e.g., set_secure_cookie() - The user object returned by get_authenticated_user() includes the - attributes 'facebook_uid' and 'name' in addition to session attributes - like 'session_key'. You should save the session key with the user; it is + The user object returned by `get_authenticated_user` includes the + attributes ``facebook_uid`` and ``name`` in addition to session attributes + like ``session_key``. You should save the session key with the user; it is required to make requests on behalf of the user later with - facebook_request(). + `facebook_request`. """ + @return_future def authenticate_redirect(self, callback_uri=None, cancel_uri=None, - extended_permissions=None): - """Authenticates/installs this app for the current user.""" + extended_permissions=None, callback=None): + """Authenticates/installs this app for the current user. + + .. versionchanged:: 3.1 + Returns a `.Future` and takes an optional callback. These are + not strictly necessary as this method is synchronous, + but they are supplied for consistency with + `OAuthMixin.authorize_redirect`. + """ self.require_setting("facebook_api_key", "Facebook Connect") callback_uri = callback_uri or self.request.uri args = { @@ -792,14 +1005,15 @@ args["cancel_url"] = urlparse.urljoin( self.request.full_url(), cancel_uri) if extended_permissions: - if isinstance(extended_permissions, (unicode, bytes_type)): + if isinstance(extended_permissions, (unicode_type, bytes_type)): extended_permissions = [extended_permissions] args["req_perms"] = ",".join(extended_permissions) self.redirect("http://www.facebook.com/login.php?" + - urllib.urlencode(args)) + urllib_parse.urlencode(args)) + callback() def authorize_redirect(self, extended_permissions, callback_uri=None, - cancel_uri=None): + cancel_uri=None, callback=None): """Redirects to an authorization request for the given FB resource. The available resource names are listed at @@ -815,9 +1029,16 @@ names. To get the session secret and session key, call get_authenticated_user() just as you would with authenticate_redirect(). + + .. versionchanged:: 3.1 + Returns a `.Future` and takes an optional callback. These are + not strictly necessary as this method is synchronous, + but they are supplied for consistency with + `OAuthMixin.authorize_redirect`. """ - self.authenticate_redirect(callback_uri, cancel_uri, - extended_permissions) + return self.authenticate_redirect(callback_uri, cancel_uri, + extended_permissions, + callback=callback) def get_authenticated_user(self, callback): """Fetches the authenticated Facebook user. @@ -834,7 +1055,7 @@ self._on_get_user_info, callback, session), session_key=session["session_key"], uids=session["uid"], - fields="uid,first_name,last_name,name,locale,pic_square," \ + fields="uid,first_name,last_name,name,locale,pic_square," "profile_url,username") def facebook_request(self, method, callback, **args): @@ -878,8 +1099,8 @@ args["format"] = "json" args["sig"] = self._signature(args) url = "http://api.facebook.com/restserver.php?" + \ - urllib.urlencode(args) - http = httpclient.AsyncHTTPClient() + urllib_parse.urlencode(args) + http = self.get_auth_http_client() http.fetch(url, callback=self.async_callback( self._parse_response, callback)) @@ -902,17 +1123,17 @@ def _parse_response(self, callback, response): if response.error: - logging.warning("HTTP error from Facebook: %s", response.error) + gen_log.warning("HTTP error from Facebook: %s", response.error) callback(None) return try: json = escape.json_decode(response.body) except Exception: - logging.warning("Invalid JSON from Facebook: %r", response.body) + gen_log.warning("Invalid JSON from Facebook: %r", response.body) callback(None) return if isinstance(json, dict) and json.get("error_code"): - logging.warning("Facebook error: %d: %r", json["error_code"], + gen_log.warning("Facebook error: %d: %r", json["error_code"], json.get("error_msg")) callback(None) return @@ -921,83 +1142,90 @@ def _signature(self, args): parts = ["%s=%s" % (n, args[n]) for n in sorted(args.keys())] body = "".join(parts) + self.settings["facebook_secret"] - if isinstance(body, unicode): body = body.encode("utf-8") + if isinstance(body, unicode_type): + body = body.encode("utf-8") return hashlib.md5(body).hexdigest() + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + + class FacebookGraphMixin(OAuth2Mixin): """Facebook authentication using the new Graph API and OAuth2.""" _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?" _OAUTH_AUTHORIZE_URL = "https://graph.facebook.com/oauth/authorize?" _OAUTH_NO_CALLBACKS = False + _FACEBOOK_BASE_URL = "https://graph.facebook.com" + @_auth_return_future def get_authenticated_user(self, redirect_uri, client_id, client_secret, - code, callback, extra_fields=None): - """Handles the login for the Facebook user, returning a user object. + code, callback, extra_fields=None): + """Handles the login for the Facebook user, returning a user object. - Example usage:: + Example usage:: - class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin): - @tornado.web.asynchronous - def get(self): - if self.get_argument("code", False): - self.get_authenticated_user( - redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - client_secret=self.settings["facebook_secret"], - code=self.get_argument("code"), - callback=self.async_callback( - self._on_login)) - return - self.authorize_redirect(redirect_uri='/auth/facebookgraph/', - client_id=self.settings["facebook_api_key"], - extra_params={"scope": "read_stream,offline_access"}) - - def _on_login(self, user): - logging.error(user) - self.finish() - - """ - http = httpclient.AsyncHTTPClient() - args = { - "redirect_uri": redirect_uri, - "code": code, - "client_id": client_id, - "client_secret": client_secret, - } - - fields = set(['id', 'name', 'first_name', 'last_name', - 'locale', 'picture', 'link']) - if extra_fields: fields.update(extra_fields) - - http.fetch(self._oauth_request_token_url(**args), - self.async_callback(self._on_access_token, redirect_uri, client_id, - client_secret, callback, fields)) + class FacebookGraphLoginHandler(LoginHandler, tornado.auth.FacebookGraphMixin): + @tornado.web.asynchronous + @tornado.gen.coroutine + def get(self): + if self.get_argument("code", False): + user = yield self.get_authenticated_user( + redirect_uri='/auth/facebookgraph/', + client_id=self.settings["facebook_api_key"], + client_secret=self.settings["facebook_secret"], + code=self.get_argument("code")) + # Save the user with e.g. set_secure_cookie + else: + yield self.authorize_redirect( + redirect_uri='/auth/facebookgraph/', + client_id=self.settings["facebook_api_key"], + extra_params={"scope": "read_stream,offline_access"}) + """ + http = self.get_auth_http_client() + args = { + "redirect_uri": redirect_uri, + "code": code, + "client_id": client_id, + "client_secret": client_secret, + } + + fields = set(['id', 'name', 'first_name', 'last_name', + 'locale', 'picture', 'link']) + if extra_fields: + fields.update(extra_fields) + + http.fetch(self._oauth_request_token_url(**args), + self.async_callback(self._on_access_token, redirect_uri, client_id, + client_secret, callback, fields)) def _on_access_token(self, redirect_uri, client_id, client_secret, - callback, fields, response): - if response.error: - logging.warning('Facebook auth error: %s' % str(response)) - callback(None) - return - - args = escape.parse_qs_bytes(escape.native_str(response.body)) - session = { - "access_token": args["access_token"][-1], - "expires": args.get("expires") - } - - self.facebook_request( - path="/me", - callback=self.async_callback( - self._on_get_user_info, callback, session, fields), - access_token=session["access_token"], - fields=",".join(fields) - ) + future, fields, response): + if response.error: + future.set_exception(AuthError('Facebook auth error: %s' % str(response))) + return + + args = escape.parse_qs_bytes(escape.native_str(response.body)) + session = { + "access_token": args["access_token"][-1], + "expires": args.get("expires") + } + self.facebook_request( + path="/me", + callback=self.async_callback( + self._on_get_user_info, future, session, fields), + access_token=session["access_token"], + fields=",".join(fields) + ) - def _on_get_user_info(self, callback, session, fields, user): + def _on_get_user_info(self, future, session, fields, user): if user is None: - callback(None) + future.set_result(None) return fieldmap = {} @@ -1005,65 +1233,82 @@ fieldmap[field] = user.get(field) fieldmap.update({"access_token": session["access_token"], "session_expires": session.get("expires")}) - callback(fieldmap) + future.set_result(fieldmap) + @_auth_return_future def facebook_request(self, path, callback, access_token=None, - post_args=None, **args): + post_args=None, **args): """Fetches the given relative API path, e.g., "/btaylor/picture" - If the request is a POST, post_args should be provided. Query + If the request is a POST, ``post_args`` should be provided. Query string arguments should be given as keyword arguments. An introduction to the Facebook Graph API can be found at http://developers.facebook.com/docs/api - Many methods require an OAuth access token which you can obtain - through authorize_redirect() and get_authenticated_user(). The - user returned through that process includes an 'access_token' - attribute that can be used to make authenticated requests via - this method. Example usage:: + Many methods require an OAuth access token which you can + obtain through `~OAuth2Mixin.authorize_redirect` and + `get_authenticated_user`. The user returned through that + process includes an ``access_token`` attribute that can be + used to make authenticated requests via this method. + + Example usage:: class MainHandler(tornado.web.RequestHandler, tornado.auth.FacebookGraphMixin): @tornado.web.authenticated @tornado.web.asynchronous + @tornado.gen.coroutine def get(self): - self.facebook_request( + new_entry = yield self.facebook_request( "/me/feed", post_args={"message": "I am posting from my Tornado application!"}, - access_token=self.current_user["access_token"], - callback=self.async_callback(self._on_post)) + access_token=self.current_user["access_token"]) - def _on_post(self, new_entry): if not new_entry: # Call failed; perhaps missing permission? - self.authorize_redirect() + yield self.authorize_redirect() return self.finish("Posted a message!") + The given path is relative to ``self._FACEBOOK_BASE_URL``, + by default "https://graph.facebook.com". + + .. versionchanged:: 3.1 + Added the ability to override ``self._FACEBOOK_BASE_URL``. """ - url = "https://graph.facebook.com" + path + url = self._FACEBOOK_BASE_URL + path all_args = {} if access_token: all_args["access_token"] = access_token all_args.update(args) - all_args.update(post_args or {}) - if all_args: url += "?" + urllib.urlencode(all_args) + + if all_args: + url += "?" + urllib_parse.urlencode(all_args) callback = self.async_callback(self._on_facebook_request, callback) - http = httpclient.AsyncHTTPClient() + http = self.get_auth_http_client() if post_args is not None: - http.fetch(url, method="POST", body=urllib.urlencode(post_args), + http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), callback=callback) else: http.fetch(url, callback=callback) - def _on_facebook_request(self, callback, response): + def _on_facebook_request(self, future, response): if response.error: - logging.warning("Error response %s fetching %s", response.error, - response.request.url) - callback(None) + future.set_exception(AuthError("Error response %s fetching %s" % + (response.error, response.request.url))) return - callback(escape.json_decode(response.body)) + + future.set_result(escape.json_decode(response.body)) + + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + def _oauth_signature(consumer_token, method, url, parameters={}, token=None): """Calculates the HMAC-SHA1 OAuth signature for the given request. @@ -1079,15 +1324,16 @@ base_elems.append(normalized_url) base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()))) - base_string = "&".join(_oauth_escape(e) for e in base_elems) + base_string = "&".join(_oauth_escape(e) for e in base_elems) key_elems = [escape.utf8(consumer_token["secret"])] key_elems.append(escape.utf8(token["secret"] if token else "")) - key = b("&").join(key_elems) + key = b"&".join(key_elems) hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) return binascii.b2a_base64(hash.digest())[:-1] + def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. @@ -1103,27 +1349,30 @@ base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()))) - base_string = "&".join(_oauth_escape(e) for e in base_elems) - key_elems = [escape.utf8(urllib.quote(consumer_token["secret"], safe='~'))] - key_elems.append(escape.utf8(urllib.quote(token["secret"], safe='~') if token else "")) - key = b("&").join(key_elems) + base_string = "&".join(_oauth_escape(e) for e in base_elems) + key_elems = [escape.utf8(urllib_parse.quote(consumer_token["secret"], safe='~'))] + key_elems.append(escape.utf8(urllib_parse.quote(token["secret"], safe='~') if token else "")) + key = b"&".join(key_elems) hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) return binascii.b2a_base64(hash.digest())[:-1] + def _oauth_escape(val): - if isinstance(val, unicode): + if isinstance(val, unicode_type): val = val.encode("utf-8") - return urllib.quote(val, safe="~") + return urllib_parse.quote(val, safe="~") def _oauth_parse_response(body): - p = escape.parse_qs(body, keep_blank_values=False) - token = dict(key=p[b("oauth_token")][0], secret=p[b("oauth_token_secret")][0]) + # I can't find an officially-defined encoding for oauth responses and + # have never seen anyone use non-ascii. Leave the response in a byte + # string for python 2, and use utf8 on python 3. + body = escape.native_str(body) + p = urlparse.parse_qs(body, keep_blank_values=False) + token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) # Add the extra parameters the Provider included to the token - special = (b("oauth_token"), b("oauth_token_secret")) + special = ("oauth_token", "oauth_token_secret") token.update((k, p[k][0]) for k in p if k not in special) return token - - diff -Nru python-tornado-2.1.0/tornado/autoreload.py python-tornado-3.1.1/tornado/autoreload.py --- python-tornado-2.1.0/tornado/autoreload.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/autoreload.py 2013-08-04 19:34:21.000000000 +0000 @@ -14,49 +14,109 @@ # License for the specific language governing permissions and limitations # under the License. -"""A module to automatically restart the server when a module is modified. +"""xAutomatically restart the server when a source file is modified. -Most applications should not call this module directly. Instead, pass the +Most applications should not access this module directly. Instead, pass the keyword argument ``debug=True`` to the `tornado.web.Application` constructor. This will enable autoreload mode as well as checking for changes to templates -and static resources. +and static resources. Note that restarting is a destructive operation +and any requests in progress will be aborted when the process restarts. -This module depends on IOLoop, so it will not work in WSGI applications -and Google AppEngine. It also will not work correctly when HTTPServer's +This module can also be used as a command-line wrapper around scripts +such as unit test runners. See the `main` method for details. + +The command-line wrapper and Application debug modes can be used together. +This combination is encouraged as the wrapper catches syntax errors and +other import-time failures, while debug mode catches changes once +the server has started. + +This module depends on `.IOLoop`, so it will not work in WSGI applications +and Google App Engine. It also will not work correctly when `.HTTPServer`'s multi-process mode is used. + +Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) +because it re-executes Python using ``sys.executable`` and ``sys.argv``. +Additionally, modifying these variables will cause reloading to behave +incorrectly. """ -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, with_statement + +import os +import sys + +# sys.path handling +# ----------------- +# +# If a module is run with "python -m", the current directory (i.e. "") +# is automatically prepended to sys.path, but not if it is run as +# "path/to/file.py". The processing for "-m" rewrites the former to +# the latter, so subsequent executions won't have the same path as the +# original. +# +# Conversely, when run as path/to/file.py, the directory containing +# file.py gets added to the path, which can cause confusion as imports +# may become relative in spite of the future import. +# +# We address the former problem by setting the $PYTHONPATH environment +# variable before re-execution so the new process will see the correct +# path. We attempt to address the latter problem when tornado.autoreload +# is run as __main__, although we can't fix the general case because +# we cannot reliably reconstruct the original command line +# (http://bugs.python.org/issue14208). + +if __name__ == "__main__": + # This sys.path manipulation must come before our imports (as much + # as possible - if we introduced a tornado.sys or tornado.os + # module we'd be in trouble), or else our imports would become + # relative again despite the future import. + # + # There is a separate __main__ block at the end of the file to call main(). + if sys.path[0] == os.path.dirname(__file__): + del sys.path[0] import functools import logging import os import pkgutil import sys +import traceback import types import subprocess +import weakref from tornado import ioloop +from tornado.log import gen_log from tornado import process +from tornado.util import exec_in try: import signal except ImportError: signal = None -def start(io_loop=None, check_time=500): - """Restarts the process automatically when a module is modified. - We run on the I/O loop, and restarting is a destructive operation, - so will terminate any pending requests. - """ - io_loop = io_loop or ioloop.IOLoop.instance() - add_reload_hook(functools.partial(_close_all_fds, io_loop)) +_watched_files = set() +_reload_hooks = [] +_reload_attempted = False +_io_loops = weakref.WeakKeyDictionary() + + +def start(io_loop=None, check_time=500): + """Begins watching source files for changes using the given `.IOLoop`. """ + io_loop = io_loop or ioloop.IOLoop.current() + if io_loop in _io_loops: + return + _io_loops[io_loop] = True + if len(_io_loops) > 1: + gen_log.warning("tornado.autoreload started more than once in the same process") + add_reload_hook(functools.partial(io_loop.close, all_fds=True)) modify_times = {} callback = functools.partial(_reload_on_update, modify_times) scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop) scheduler.start() + def wait(): """Wait for a watched file to change, then restart the process. @@ -68,7 +128,6 @@ start(io_loop) io_loop.start() -_watched_files = set() def watch(filename): """Add a file to the watch list. @@ -77,26 +136,17 @@ """ _watched_files.add(filename) -_reload_hooks = [] def add_reload_hook(fn): """Add a function to be called before reloading the process. Note that for open file and socket handles it is generally preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or - `tornado.platform.auto.set_close_exec`) instead of using a reload - hook to close them. + ``tornado.platform.auto.set_close_exec``) instead + of using a reload hook to close them. """ _reload_hooks.append(fn) -def _close_all_fds(io_loop): - for fd in io_loop._handlers.keys(): - try: - os.close(fd) - except Exception: - pass - -_reload_attempted = False def _reload_on_update(modify_times): if _reload_attempted: @@ -112,15 +162,18 @@ # in the standard library), and occasionally this can cause strange # failures in getattr. Just ignore anything that's not an ordinary # module. - if not isinstance(module, types.ModuleType): continue + if not isinstance(module, types.ModuleType): + continue path = getattr(module, "__file__", None) - if not path: continue + if not path: + continue if path.endswith(".pyc") or path.endswith(".pyo"): path = path[:-1] _check_file(modify_times, path) for path in _watched_files: _check_file(modify_times, path) + def _check_file(modify_times, path): try: modified = os.stat(path).st_mtime @@ -130,9 +183,10 @@ modify_times[path] = modified return if modify_times[path] != modified: - logging.info("%s modified; restarting server", path) + gen_log.info("%s modified; restarting server", path) _reload() + def _reload(): global _reload_attempted _reload_attempted = True @@ -143,6 +197,15 @@ # ioloop.set_blocking_log_threshold so it doesn't fire # after the exec. signal.setitimer(signal.ITIMER_REAL, 0, 0) + # sys.path fixes: see comments at top of file. If sys.path[0] is an empty + # string, we were (probably) invoked with -m and the effective path + # is about to change on re-exec. Add the current directory to $PYTHONPATH + # to ensure that the new process sees the same path we did. + path_prefix = '.' + os.pathsep + if (sys.path[0] == '' and + not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): + os.environ["PYTHONPATH"] = (path_prefix + + os.environ.get("PYTHONPATH", "")) if sys.platform == 'win32': # os.execv is broken on Windows and can't properly parse command line # arguments and executable name if they contain whitespaces. subprocess @@ -173,9 +236,11 @@ python -m tornado.autoreload -m module.to.run [args...] python -m tornado.autoreload path/to/script.py [args...] """ + + def main(): """Command-line wrapper to re-run a script whenever its source changes. - + Scripts may be specified by filename or module name:: python -m tornado.autoreload -m tornado.test.runtests @@ -197,7 +262,7 @@ script = sys.argv[1] sys.argv = sys.argv[1:] else: - print >>sys.stderr, _USAGE + print(_USAGE, file=sys.stderr) sys.exit(1) try: @@ -211,40 +276,41 @@ # Use globals as our "locals" dictionary so that # something that tries to import __main__ (e.g. the unittest # module) will see the right things. - exec f.read() in globals(), globals() - except SystemExit, e: - logging.info("Script exited with status %s", e.code) - except Exception, e: - logging.warning("Script exited with uncaught exception", exc_info=True) + exec_in(f.read(), globals(), globals()) + except SystemExit as e: + logging.basicConfig() + gen_log.info("Script exited with status %s", e.code) + except Exception as e: + logging.basicConfig() + gen_log.warning("Script exited with uncaught exception", exc_info=True) + # If an exception occurred at import time, the file with the error + # never made it into sys.modules and so we won't know to watch it. + # Just to make sure we've covered everything, walk the stack trace + # from the exception and watch every file. + for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]): + watch(filename) if isinstance(e, SyntaxError): + # SyntaxErrors are special: their innermost stack frame is fake + # so extract_tb won't see it and we have to get the filename + # from the exception object. watch(e.filename) else: - logging.info("Script exited normally") + logging.basicConfig() + gen_log.info("Script exited normally") # restore sys.argv so subsequent executions will include autoreload sys.argv = original_argv if mode == 'module': # runpy did a fake import of the module as __main__, but now it's # no longer in sys.modules. Figure out where it is and watch it. - watch(pkgutil.get_loader(module).get_filename()) + loader = pkgutil.get_loader(module) + if loader is not None: + watch(loader.get_filename()) wait() - + if __name__ == "__main__": - # If this module is run with "python -m tornado.autoreload", the current - # directory is automatically prepended to sys.path, but not if it is - # run as "path/to/tornado/autoreload.py". The processing for "-m" rewrites - # the former to the latter, so subsequent executions won't have the same - # path as the original. Modify os.environ here to ensure that the - # re-executed process will have the same path. - # Conversely, when run as path/to/tornado/autoreload.py, the directory - # containing autoreload.py gets added to the path, but we don't want - # tornado modules importable at top level, so remove it. - path_prefix = '.' + os.pathsep - if (sys.path[0] == '' and - not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): - os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "") - elif sys.path[0] == os.path.dirname(__file__): - del sys.path[0] + # See also the other __main__ block at the top of the file, which modifies + # sys.path before our imports main() diff -Nru python-tornado-2.1.0/tornado/concurrent.py python-tornado-3.1.1/tornado/concurrent.py --- python-tornado-2.1.0/tornado/concurrent.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/concurrent.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,265 @@ +#!/usr/bin/env python +# +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Utilities for working with threads and ``Futures``. + +``Futures`` are a pattern for concurrent programming introduced in +Python 3.2 in the `concurrent.futures` package (this package has also +been backported to older versions of Python and can be installed with +``pip install futures``). Tornado will use `concurrent.futures.Future` if +it is available; otherwise it will use a compatible class defined in this +module. +""" +from __future__ import absolute_import, division, print_function, with_statement + +import functools +import sys + +from tornado.stack_context import ExceptionStackContext, wrap +from tornado.util import raise_exc_info, ArgReplacer + +try: + from concurrent import futures +except ImportError: + futures = None + + +class ReturnValueIgnoredError(Exception): + pass + + +class _DummyFuture(object): + def __init__(self): + self._done = False + self._result = None + self._exception = None + self._callbacks = [] + + def cancel(self): + return False + + def cancelled(self): + return False + + def running(self): + return not self._done + + def done(self): + return self._done + + def result(self, timeout=None): + self._check_done() + if self._exception: + raise self._exception + return self._result + + def exception(self, timeout=None): + self._check_done() + if self._exception: + return self._exception + else: + return None + + def add_done_callback(self, fn): + if self._done: + fn(self) + else: + self._callbacks.append(fn) + + def set_result(self, result): + self._result = result + self._set_done() + + def set_exception(self, exception): + self._exception = exception + self._set_done() + + def _check_done(self): + if not self._done: + raise Exception("DummyFuture does not support blocking for results") + + def _set_done(self): + self._done = True + for cb in self._callbacks: + # TODO: error handling + cb(self) + self._callbacks = None + +if futures is None: + Future = _DummyFuture +else: + Future = futures.Future + + +class TracebackFuture(Future): + """Subclass of `Future` which can store a traceback with + exceptions. + + The traceback is automatically available in Python 3, but in the + Python 2 futures backport this information is discarded. + """ + def __init__(self): + super(TracebackFuture, self).__init__() + self.__exc_info = None + + def exc_info(self): + return self.__exc_info + + def set_exc_info(self, exc_info): + """Traceback-aware replacement for + `~concurrent.futures.Future.set_exception`. + """ + self.__exc_info = exc_info + self.set_exception(exc_info[1]) + + def result(self): + if self.__exc_info is not None: + raise_exc_info(self.__exc_info) + else: + return super(TracebackFuture, self).result() + + +class DummyExecutor(object): + def submit(self, fn, *args, **kwargs): + future = TracebackFuture() + try: + future.set_result(fn(*args, **kwargs)) + except Exception: + future.set_exc_info(sys.exc_info()) + return future + + def shutdown(self, wait=True): + pass + +dummy_executor = DummyExecutor() + + +def run_on_executor(fn): + """Decorator to run a synchronous method asynchronously on an executor. + + The decorated method may be called with a ``callback`` keyword + argument and returns a future. + """ + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + callback = kwargs.pop("callback", None) + future = self.executor.submit(fn, self, *args, **kwargs) + if callback: + self.io_loop.add_future(future, + lambda future: callback(future.result())) + return future + return wrapper + + +_NO_RESULT = object() + + +def return_future(f): + """Decorator to make a function that returns via callback return a + `Future`. + + The wrapped function should take a ``callback`` keyword argument + and invoke it with one argument when it has finished. To signal failure, + the function can simply raise an exception (which will be + captured by the `.StackContext` and passed along to the ``Future``). + + From the caller's perspective, the callback argument is optional. + If one is given, it will be invoked when the function is complete + with `Future.result()` as an argument. If the function fails, the + callback will not be run and an exception will be raised into the + surrounding `.StackContext`. + + If no callback is given, the caller should use the ``Future`` to + wait for the function to complete (perhaps by yielding it in a + `.gen.engine` function, or passing it to `.IOLoop.add_future`). + + Usage:: + + @return_future + def future_func(arg1, arg2, callback): + # Do stuff (possibly asynchronous) + callback(result) + + @gen.engine + def caller(callback): + yield future_func(arg1, arg2) + callback() + + Note that ``@return_future`` and ``@gen.engine`` can be applied to the + same function, provided ``@return_future`` appears first. However, + consider using ``@gen.coroutine`` instead of this combination. + """ + replacer = ArgReplacer(f, 'callback') + + @functools.wraps(f) + def wrapper(*args, **kwargs): + future = TracebackFuture() + callback, args, kwargs = replacer.replace( + lambda value=_NO_RESULT: future.set_result(value), + args, kwargs) + + def handle_error(typ, value, tb): + future.set_exc_info((typ, value, tb)) + return True + exc_info = None + with ExceptionStackContext(handle_error): + try: + result = f(*args, **kwargs) + if result is not None: + raise ReturnValueIgnoredError( + "@return_future should not be used with functions " + "that return values") + except: + exc_info = sys.exc_info() + raise + if exc_info is not None: + # If the initial synchronous part of f() raised an exception, + # go ahead and raise it to the caller directly without waiting + # for them to inspect the Future. + raise_exc_info(exc_info) + + # If the caller passed in a callback, schedule it to be called + # when the future resolves. It is important that this happens + # just before we return the future, or else we risk confusing + # stack contexts with multiple exceptions (one here with the + # immediate exception, and again when the future resolves and + # the callback triggers its exception by calling future.result()). + if callback is not None: + def run_callback(future): + result = future.result() + if result is _NO_RESULT: + callback() + else: + callback(future.result()) + future.add_done_callback(wrap(run_callback)) + return future + return wrapper + + +def chain_future(a, b): + """Chain two futures together so that when one completes, so does the other. + + The result (success or failure) of ``a`` will be copied to ``b``. + """ + def copy(future): + assert future is a + if (isinstance(a, TracebackFuture) and isinstance(b, TracebackFuture) + and a.exc_info() is not None): + b.set_exc_info(a.exc_info()) + elif a.exception() is not None: + b.set_exception(a.exception()) + else: + b.set_result(a.result()) + a.add_done_callback(copy) diff -Nru python-tornado-2.1.0/tornado/curl_httpclient.py python-tornado-3.1.1/tornado/curl_httpclient.py --- python-tornado-2.1.0/tornado/curl_httpclient.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/curl_httpclient.py 2013-08-04 19:34:21.000000000 +0000 @@ -14,11 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -"""Blocking and non-blocking HTTP client implementations using pycurl.""" +"""Non-blocking HTTP client implementation using pycurl.""" -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, with_statement -import cStringIO import collections import logging import pycurl @@ -27,20 +26,26 @@ from tornado import httputil from tornado import ioloop +from tornado.log import gen_log from tornado import stack_context -from tornado.escape import utf8 -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.escape import utf8, native_str +from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.util import bytes_type + +try: + from io import BytesIO # py3 +except ImportError: + from cStringIO import StringIO as BytesIO # py2 + class CurlAsyncHTTPClient(AsyncHTTPClient): - def initialize(self, io_loop=None, max_clients=10, - max_simultaneous_connections=None): - self.io_loop = io_loop + def initialize(self, io_loop, max_clients=10, defaults=None): + super(CurlAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) - self._curls = [_curl_create(max_simultaneous_connections) - for i in xrange(max_clients)] + self._curls = [_curl_create() for i in range(max_clients)] self._free_list = self._curls[:] self._requests = collections.deque() self._fds = {} @@ -52,7 +57,7 @@ # socket_action is found in pycurl since 7.18.2 (it's been # in libcurl longer than that but wasn't accessible to # python). - logging.warning("socket_action method missing from pycurl; " + gen_log.warning("socket_action method missing from pycurl; " "falling back to socket_all. Upgrading " "libcurl and pycurl will improve performance") self._socket_action = \ @@ -66,18 +71,27 @@ self._handle_force_timeout, 1000, io_loop=io_loop) self._force_timeout_callback.start() + # Work around a bug in libcurl 7.29.0: Some fields in the curl + # multi object are initialized lazily, and its destructor will + # segfault if it is destroyed without having been used. Add + # and remove a dummy handle to make sure everything is + # initialized. + dummy_curl_handle = pycurl.Curl() + self._multi.add_handle(dummy_curl_handle) + self._multi.remove_handle(dummy_curl_handle) + def close(self): self._force_timeout_callback.stop() + if self._timeout is not None: + self.io_loop.remove_timeout(self._timeout) for curl in self._curls: curl.close() self._multi.close() self._closed = True super(CurlAsyncHTTPClient, self).close() - def fetch(self, request, callback, **kwargs): - if not isinstance(request, HTTPRequest): - request = HTTPRequest(url=request, **kwargs) - self._requests.append((request, stack_context.wrap(callback))) + def fetch_impl(self, request, callback): + self._requests.append((request, callback)) self._process_queue() self._set_timeout(0) @@ -92,36 +106,45 @@ pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE } if event == pycurl.POLL_REMOVE: - self.io_loop.remove_handler(fd) - del self._fds[fd] + if fd in self._fds: + self.io_loop.remove_handler(fd) + del self._fds[fd] else: ioloop_event = event_map[event] - if fd not in self._fds: - self._fds[fd] = ioloop_event - self.io_loop.add_handler(fd, self._handle_events, - ioloop_event) - else: - self._fds[fd] = ioloop_event - self.io_loop.update_handler(fd, ioloop_event) + # libcurl sometimes closes a socket and then opens a new + # one using the same FD without giving us a POLL_NONE in + # between. This is a problem with the epoll IOLoop, + # because the kernel can tell when a socket is closed and + # removes it from the epoll automatically, causing future + # update_handler calls to fail. Since we can't tell when + # this has happened, always use remove and re-add + # instead of update. + if fd in self._fds: + self.io_loop.remove_handler(fd) + self.io_loop.add_handler(fd, self._handle_events, + ioloop_event) + self._fds[fd] = ioloop_event def _set_timeout(self, msecs): """Called by libcurl to schedule a timeout.""" if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = self.io_loop.add_timeout( - time.time() + msecs/1000.0, self._handle_timeout) + self.io_loop.time() + msecs / 1000.0, self._handle_timeout) def _handle_events(self, fd, events): """Called by IOLoop when there is activity on one of our file descriptors. """ action = 0 - if events & ioloop.IOLoop.READ: action |= pycurl.CSELECT_IN - if events & ioloop.IOLoop.WRITE: action |= pycurl.CSELECT_OUT + if events & ioloop.IOLoop.READ: + action |= pycurl.CSELECT_IN + if events & ioloop.IOLoop.WRITE: + action |= pycurl.CSELECT_OUT while True: try: ret, num_handles = self._socket_action(fd, action) - except pycurl.error, e: + except pycurl.error as e: ret = e.args[0] if ret != pycurl.E_CALL_MULTI_PERFORM: break @@ -135,7 +158,7 @@ try: ret, num_handles = self._socket_action( pycurl.SOCKET_TIMEOUT, 0) - except pycurl.error, e: + except pycurl.error as e: ret = e.args[0] if ret != pycurl.E_CALL_MULTI_PERFORM: break @@ -155,7 +178,7 @@ # libcurl is ready. After each timeout, resync the scheduled # timeout with libcurl's current state. new_timeout = self._multi.timeout() - if new_timeout != -1: + if new_timeout >= 0: self._set_timeout(new_timeout) def _handle_force_timeout(self): @@ -166,7 +189,7 @@ while True: try: ret, num_handles = self._multi.socket_all() - except pycurl.error, e: + except pycurl.error as e: ret = e.args[0] if ret != pycurl.E_CALL_MULTI_PERFORM: break @@ -196,7 +219,7 @@ (request, callback) = self._requests.popleft() curl.info = { "headers": httputil.HTTPHeaders(), - "buffer": cStringIO.StringIO(), + "buffer": BytesIO(), "request": request, "callback": callback, "curl_start_time": time.time(), @@ -240,7 +263,7 @@ starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), - ) + ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], @@ -250,7 +273,6 @@ except Exception: self.handle_callback_exception(info["callback"]) - def handle_callback_exception(self, callback): self.io_loop.handle_callback_exception(callback) @@ -261,17 +283,16 @@ self.errno = errno -def _curl_create(max_simultaneous_connections=None): +def _curl_create(): curl = pycurl.Curl() - if logging.getLogger().isEnabledFor(logging.DEBUG): + if gen_log.isEnabledFor(logging.DEBUG): curl.setopt(pycurl.VERBOSE, 1) curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) - curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5) return curl def _curl_setup_request(curl, request, buffer, headers): - curl.setopt(pycurl.URL, request.url) + curl.setopt(pycurl.URL, native_str(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, @@ -291,10 +312,10 @@ # Request headers may be either a regular dict or HTTPHeaders object if isinstance(request.headers, httputil.HTTPHeaders): curl.setopt(pycurl.HTTPHEADER, - [utf8("%s: %s" % i) for i in request.headers.get_all()]) + [native_str("%s: %s" % i) for i in request.headers.get_all()]) else: curl.setopt(pycurl.HTTPHEADER, - [utf8("%s: %s" % i) for i in request.headers.iteritems()]) + [native_str("%s: %s" % i) for i in request.headers.items()]) if request.header_callback: curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) @@ -302,15 +323,26 @@ curl.setopt(pycurl.HEADERFUNCTION, lambda line: _curl_header_callback(headers, line)) if request.streaming_callback: - curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) + write_function = request.streaming_callback else: - curl.setopt(pycurl.WRITEFUNCTION, buffer.write) + write_function = buffer.write + if bytes_type is str: # py2 + curl.setopt(pycurl.WRITEFUNCTION, write_function) + else: # py3 + # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes + # a fork/port. That version has a bug in which it passes unicode + # strings instead of bytes to the WRITEFUNCTION. This means that + # if you use a WRITEFUNCTION (which tornado always does), you cannot + # download arbitrary binary data. This needs to be fixed in the + # ported pycurl package, but in the meantime this lambda will + # make it work for downloading (utf8) text. + curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s))) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) - curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout)) - curl.setopt(pycurl.TIMEOUT, int(request.request_timeout)) + curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) + curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: - curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) + curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: @@ -324,7 +356,7 @@ curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: credentials = '%s:%s' % (request.proxy_username, - request.proxy_password) + request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') @@ -351,7 +383,7 @@ # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) - # Set the request method through curl's retarded interface which makes + # Set the request method through curl's irritating interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, @@ -359,7 +391,7 @@ "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } - custom_methods = set(["DELETE"]) + custom_methods = set(["DELETE", "OPTIONS", "PATCH"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: @@ -372,7 +404,7 @@ # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): - request_buffer = cStringIO.StringIO(utf8(request.body)) + request_buffer = BytesIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": def ioctl(cmd): @@ -383,18 +415,28 @@ else: curl.setopt(pycurl.INFILESIZE, len(request.body)) - if request.auth_username and request.auth_password: - userpwd = "%s:%s" % (request.auth_username, request.auth_password) - curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) - curl.setopt(pycurl.USERPWD, userpwd) - logging.debug("%s %s (username: %r)", request.method, request.url, + if request.auth_username is not None: + userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') + + if request.auth_mode is None or request.auth_mode == "basic": + curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) + elif request.auth_mode == "digest": + curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) + else: + raise ValueError("Unsupported auth_mode %s" % request.auth_mode) + + curl.setopt(pycurl.USERPWD, native_str(userpwd)) + gen_log.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: curl.unsetopt(pycurl.USERPWD) - logging.debug("%s %s", request.method, request.url) + gen_log.debug("%s %s", request.method, request.url) - if request.client_key is not None or request.client_cert is not None: - raise ValueError("Client certificate not supported with curl_httpclient") + if request.client_cert is not None: + curl.setopt(pycurl.SSLCERT, request.client_cert) + + if request.client_key is not None: + curl.setopt(pycurl.SSLKEY, request.client_key) if threading.activeCount() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads @@ -420,15 +462,16 @@ return headers.parse_line(header_line) + def _curl_debug(debug_type, debug_msg): debug_types = ('I', '<', '>', '<', '>') if debug_type == 0: - logging.debug('%s', debug_msg.strip()) + gen_log.debug('%s', debug_msg.strip()) elif debug_type in (1, 2): for line in debug_msg.splitlines(): - logging.debug('%s %s', debug_types[debug_type], line) + gen_log.debug('%s %s', debug_types[debug_type], line) elif debug_type == 4: - logging.debug('%s %r', debug_types[debug_type], debug_msg) + gen_log.debug('%s %r', debug_types[debug_type], debug_msg) if __name__ == "__main__": AsyncHTTPClient.configure(CurlAsyncHTTPClient) diff -Nru python-tornado-2.1.0/tornado/database.py python-tornado-3.1.1/tornado/database.py --- python-tornado-2.1.0/tornado/database.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/database.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,229 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A lightweight wrapper around MySQLdb.""" - -import copy -import MySQLdb.constants -import MySQLdb.converters -import MySQLdb.cursors -import itertools -import logging -import time - -class Connection(object): - """A lightweight wrapper around MySQLdb DB-API connections. - - The main value we provide is wrapping rows in a dict/object so that - columns can be accessed by name. Typical usage:: - - db = database.Connection("localhost", "mydatabase") - for article in db.query("SELECT * FROM articles"): - print article.title - - Cursors are hidden by the implementation, but other than that, the methods - are very similar to the DB-API. - - We explicitly set the timezone to UTC and the character encoding to - UTF-8 on all connections to avoid time zone and encoding errors. - """ - def __init__(self, host, database, user=None, password=None, - max_idle_time=7*3600): - self.host = host - self.database = database - self.max_idle_time = max_idle_time - - args = dict(conv=CONVERSIONS, use_unicode=True, charset="utf8", - db=database, init_command='SET time_zone = "+0:00"', - sql_mode="TRADITIONAL") - if user is not None: - args["user"] = user - if password is not None: - args["passwd"] = password - - # We accept a path to a MySQL socket file or a host(:port) string - if "/" in host: - args["unix_socket"] = host - else: - self.socket = None - pair = host.split(":") - if len(pair) == 2: - args["host"] = pair[0] - args["port"] = int(pair[1]) - else: - args["host"] = host - args["port"] = 3306 - - self._db = None - self._db_args = args - self._last_use_time = time.time() - try: - self.reconnect() - except Exception: - logging.error("Cannot connect to MySQL on %s", self.host, - exc_info=True) - - def __del__(self): - self.close() - - def close(self): - """Closes this database connection.""" - if getattr(self, "_db", None) is not None: - self._db.close() - self._db = None - - def reconnect(self): - """Closes the existing database connection and re-opens it.""" - self.close() - self._db = MySQLdb.connect(**self._db_args) - self._db.autocommit(True) - - def iter(self, query, *parameters): - """Returns an iterator for the given query and parameters.""" - self._ensure_connected() - cursor = MySQLdb.cursors.SSCursor(self._db) - try: - self._execute(cursor, query, parameters) - column_names = [d[0] for d in cursor.description] - for row in cursor: - yield Row(zip(column_names, row)) - finally: - cursor.close() - - def query(self, query, *parameters): - """Returns a row list for the given query and parameters.""" - cursor = self._cursor() - try: - self._execute(cursor, query, parameters) - column_names = [d[0] for d in cursor.description] - return [Row(itertools.izip(column_names, row)) for row in cursor] - finally: - cursor.close() - - def get(self, query, *parameters): - """Returns the first row returned for the given query.""" - rows = self.query(query, *parameters) - if not rows: - return None - elif len(rows) > 1: - raise Exception("Multiple rows returned for Database.get() query") - else: - return rows[0] - - # rowcount is a more reasonable default return value than lastrowid, - # but for historical compatibility execute() must return lastrowid. - def execute(self, query, *parameters): - """Executes the given query, returning the lastrowid from the query.""" - return self.execute_lastrowid(query, *parameters) - - def execute_lastrowid(self, query, *parameters): - """Executes the given query, returning the lastrowid from the query.""" - cursor = self._cursor() - try: - self._execute(cursor, query, parameters) - return cursor.lastrowid - finally: - cursor.close() - - def execute_rowcount(self, query, *parameters): - """Executes the given query, returning the rowcount from the query.""" - cursor = self._cursor() - try: - self._execute(cursor, query, parameters) - return cursor.rowcount - finally: - cursor.close() - - def executemany(self, query, parameters): - """Executes the given query against all the given param sequences. - - We return the lastrowid from the query. - """ - return self.executemany_lastrowid(query, parameters) - - def executemany_lastrowid(self, query, parameters): - """Executes the given query against all the given param sequences. - - We return the lastrowid from the query. - """ - cursor = self._cursor() - try: - cursor.executemany(query, parameters) - return cursor.lastrowid - finally: - cursor.close() - - def executemany_rowcount(self, query, parameters): - """Executes the given query against all the given param sequences. - - We return the rowcount from the query. - """ - cursor = self._cursor() - try: - cursor.executemany(query, parameters) - return cursor.rowcount - finally: - cursor.close() - - def _ensure_connected(self): - # Mysql by default closes client connections that are idle for - # 8 hours, but the client library does not report this fact until - # you try to perform a query and it fails. Protect against this - # case by preemptively closing and reopening the connection - # if it has been idle for too long (7 hours by default). - if (self._db is None or - (time.time() - self._last_use_time > self.max_idle_time)): - self.reconnect() - self._last_use_time = time.time() - - def _cursor(self): - self._ensure_connected() - return self._db.cursor() - - def _execute(self, cursor, query, parameters): - try: - return cursor.execute(query, parameters) - except OperationalError: - logging.error("Error connecting to MySQL on %s", self.host) - self.close() - raise - - -class Row(dict): - """A dict that allows for object-like property access syntax.""" - def __getattr__(self, name): - try: - return self[name] - except KeyError: - raise AttributeError(name) - - -# Fix the access conversions to properly recognize unicode/binary -FIELD_TYPE = MySQLdb.constants.FIELD_TYPE -FLAG = MySQLdb.constants.FLAG -CONVERSIONS = copy.copy(MySQLdb.converters.conversions) - -field_types = [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING] -if 'VARCHAR' in vars(FIELD_TYPE): - field_types.append(FIELD_TYPE.VARCHAR) - -for field_type in field_types: - CONVERSIONS[field_type] = [(FLAG.BINARY, str)] + CONVERSIONS[field_type] - - -# Alias some common MySQL exceptions -IntegrityError = MySQLdb.IntegrityError -OperationalError = MySQLdb.OperationalError diff -Nru python-tornado-2.1.0/tornado/epoll.c python-tornado-3.1.1/tornado/epoll.c --- python-tornado-2.1.0/tornado/epoll.c 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/epoll.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,112 +0,0 @@ -/* - * Copyright 2009 Facebook - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -#include "Python.h" -#include -#include - -#define MAX_EVENTS 24 - -/* - * Simple wrapper around epoll_create. - */ -static PyObject* _epoll_create(void) { - int fd = epoll_create(MAX_EVENTS); - if (fd == -1) { - PyErr_SetFromErrno(PyExc_Exception); - return NULL; - } - - return PyInt_FromLong(fd); -} - -/* - * Simple wrapper around epoll_ctl. We throw an exception if the call fails - * rather than returning the error code since it is an infrequent (and likely - * catastrophic) event when it does happen. - */ -static PyObject* _epoll_ctl(PyObject* self, PyObject* args) { - int epfd, op, fd, events; - struct epoll_event event; - - if (!PyArg_ParseTuple(args, "iiiI", &epfd, &op, &fd, &events)) { - return NULL; - } - - memset(&event, 0, sizeof(event)); - event.events = events; - event.data.fd = fd; - if (epoll_ctl(epfd, op, fd, &event) == -1) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } - - Py_INCREF(Py_None); - return Py_None; -} - -/* - * Simple wrapper around epoll_wait. We return None if the call times out and - * throw an exception if an error occurs. Otherwise, we return a list of - * (fd, event) tuples. - */ -static PyObject* _epoll_wait(PyObject* self, PyObject* args) { - struct epoll_event events[MAX_EVENTS]; - int epfd, timeout, num_events, i; - PyObject* list; - PyObject* tuple; - - if (!PyArg_ParseTuple(args, "ii", &epfd, &timeout)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - num_events = epoll_wait(epfd, events, MAX_EVENTS, timeout); - Py_END_ALLOW_THREADS - if (num_events == -1) { - PyErr_SetFromErrno(PyExc_Exception); - return NULL; - } - - list = PyList_New(num_events); - for (i = 0; i < num_events; i++) { - tuple = PyTuple_New(2); - PyTuple_SET_ITEM(tuple, 0, PyInt_FromLong(events[i].data.fd)); - PyTuple_SET_ITEM(tuple, 1, PyInt_FromLong(events[i].events)); - PyList_SET_ITEM(list, i, tuple); - } - return list; -} - -/* - * Our method declararations - */ -static PyMethodDef kEpollMethods[] = { - {"epoll_create", (PyCFunction)_epoll_create, METH_NOARGS, - "Create an epoll file descriptor"}, - {"epoll_ctl", _epoll_ctl, METH_VARARGS, - "Control an epoll file descriptor"}, - {"epoll_wait", _epoll_wait, METH_VARARGS, - "Wait for events on an epoll file descriptor"}, - {NULL, NULL, 0, NULL} -}; - -/* - * Module initialization - */ -PyMODINIT_FUNC initepoll(void) { - Py_InitModule("epoll", kEpollMethods); -} diff -Nru python-tornado-2.1.0/tornado/escape.py python-tornado-3.1.1/tornado/escape.py --- python-tornado-2.1.0/tornado/escape.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/escape.py 2013-09-01 18:41:35.000000000 +0000 @@ -20,50 +20,41 @@ have crept in over time. """ -import htmlentitydefs +from __future__ import absolute_import, division, print_function, with_statement + import re import sys -import urllib -# Python3 compatibility: On python2.5, introduce the bytes alias from 2.6 -try: bytes -except Exception: bytes = str +from tornado.util import bytes_type, unicode_type, basestring_type, u try: - from urlparse import parse_qs # Python 2.6+ + from urllib.parse import parse_qs as _parse_qs # py3 except ImportError: - from cgi import parse_qs + from urlparse import parse_qs as _parse_qs # Python 2.6+ -# json module is in the standard library as of python 2.6; fall back to -# simplejson if present for older versions. try: - import json - assert hasattr(json, "loads") and hasattr(json, "dumps") - _json_decode = json.loads - _json_encode = json.dumps -except Exception: - try: - import simplejson - _json_decode = lambda s: simplejson.loads(_unicode(s)) - _json_encode = lambda v: simplejson.dumps(v) - except ImportError: - try: - # For Google AppEngine - from django.utils import simplejson - _json_decode = lambda s: simplejson.loads(_unicode(s)) - _json_encode = lambda v: simplejson.dumps(v) - except ImportError: - def _json_decode(s): - raise NotImplementedError( - "A JSON parser is required, e.g., simplejson at " - "http://pypi.python.org/pypi/simplejson/") - _json_encode = _json_decode + import htmlentitydefs # py2 +except ImportError: + import html.entities as htmlentitydefs # py3 + +try: + import urllib.parse as urllib_parse # py3 +except ImportError: + import urllib as urllib_parse # py2 +import json + +try: + unichr +except NameError: + unichr = chr _XHTML_ESCAPE_RE = re.compile('[&<>"]') _XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"'} + + def xhtml_escape(value): - """Escapes a string so it is valid within XML or XHTML.""" + """Escapes a string so it is valid within HTML or XML.""" return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value)) @@ -73,6 +64,9 @@ return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) +# The fact that json_encode wraps json.dumps is an implementation detail. +# Please see https://github.com/facebook/tornado/pull/706 +# before sending a pull request that adds **kwargs to this function. def json_encode(value): """JSON-encodes the given Python object.""" # JSON permits but does not require forward slashes to be escaped. @@ -81,12 +75,12 @@ # the javscript. Some json libraries do this escaping by default, # although python's standard library does not, so we do it here. # http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped - return _json_encode(recursive_unicode(value)).replace("?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""") +# Use to_unicode instead of tornado.util.u - we don't want backslashes getting +# processed as escapes. +_URL_RE = re.compile(to_unicode(r"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""")) def linkify(text, shorten=False, extra_params="", @@ -232,19 +272,29 @@ Parameters: - shorten: Long urls will be shortened for display. + * ``shorten``: Long urls will be shortened for display. - extra_params: Extra text to include in the link tag, - e.g. linkify(text, extra_params='rel="nofollow" class="external"') + * ``extra_params``: Extra text to include in the link tag, or a callable + taking the link as an argument and returning the extra text + e.g. ``linkify(text, extra_params='rel="nofollow" class="external"')``, + or:: + + def extra_params_cb(url): + if url.startswith("http://example.com"): + return 'class="internal"' + else: + return 'class="external" rel="nofollow"' + linkify(text, extra_params=extra_params_cb) - require_protocol: Only linkify urls which include a protocol. If this is - False, urls such as www.facebook.com will also be linkified. + * ``require_protocol``: Only linkify urls which include a protocol. If + this is False, urls such as www.facebook.com will also be linkified. - permitted_protocols: List (or set) of protocols which should be linkified, - e.g. linkify(text, permitted_protocols=["http", "ftp", "mailto"]). - It is very unsafe to include protocols such as "javascript". + * ``permitted_protocols``: List (or set) of protocols which should be + linkified, e.g. ``linkify(text, permitted_protocols=["http", "ftp", + "mailto"])``. It is very unsafe to include protocols such as + ``javascript``. """ - if extra_params: + if extra_params and not callable(extra_params): extra_params = " " + extra_params.strip() def make_link(m): @@ -260,7 +310,10 @@ if not proto: href = "http://" + href # no proto specified, use http - params = extra_params + if callable(extra_params): + params = " " + extra_params(href).strip() + else: + params = extra_params # clip long urls. max_len is just an approximation max_len = 30 @@ -278,7 +331,7 @@ # (no more slug, etc), so it really just provides a little # extra indication of shortening. url = url[:proto_len] + parts[0] + "/" + \ - parts[1][:8].split('?')[0].split('.')[0] + parts[1][:8].split('?')[0].split('.')[0] if len(url) > max_len * 1.5: # still too long url = url[:max_len] @@ -297,7 +350,7 @@ # have a status bar, such as Safari by default) params += ' title="%s"' % href - return u'%s' % (href, params, url) + return u('%s') % (href, params, url) # First HTML-escape so that our strings are all safe. # The regex is modified to avoid character entites other than & so @@ -320,7 +373,7 @@ def _build_unicode_map(): unicode_map = {} - for name, value in htmlentitydefs.name2codepoint.iteritems(): + for name, value in htmlentitydefs.name2codepoint.items(): unicode_map[name] = unichr(value) return unicode_map diff -Nru python-tornado-2.1.0/tornado/gen.py python-tornado-3.1.1/tornado/gen.py --- python-tornado-2.1.0/tornado/gen.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/gen.py 2013-08-04 19:34:21.000000000 +0000 @@ -19,30 +19,41 @@ could be written with ``gen`` as:: class GenAsyncHandler(RequestHandler): - @asynchronous - @gen.engine + @gen.coroutine def get(self): http_client = AsyncHTTPClient() - response = yield gen.Task(http_client.fetch, "http://example.com") + response = yield http_client.fetch("http://example.com") do_something_with_response(response) self.render("template.html") -`Task` works with any function that takes a ``callback`` keyword -argument. You can also yield a list of ``Tasks``, which will be +Most asynchronous functions in Tornado return a `.Future`; +yielding this object returns its `~.Future.result`. + +For functions that do not return ``Futures``, `Task` works with any +function that takes a ``callback`` keyword argument (most Tornado functions +can be used in either style, although the ``Future`` style is preferred +since it is both shorter and provides better exception handling):: + + @gen.coroutine + def get(self): + yield gen.Task(AsyncHTTPClient().fetch, "http://example.com") + +You can also yield a list of ``Futures`` and/or ``Tasks``, which will be started at the same time and run in parallel; a list of results will be returned when they are all finished:: + @gen.coroutine def get(self): http_client = AsyncHTTPClient() - response1, response2 = yield [gen.Task(http_client.fetch, url1), - gen.Task(http_client.fetch, url2)] + response1, response2 = yield [http_client.fetch(url1), + http_client.fetch(url2)] For more complicated interfaces, `Task` can be split into two parts: `Callback` and `Wait`:: class GenAsyncHandler2(RequestHandler): @asynchronous - @gen.engine + @gen.coroutine def get(self): http_client = AsyncHTTPClient() http_client.fetch("http://example.com", @@ -62,43 +73,193 @@ called with more than one argument or any keyword arguments, the result is an `Arguments` object, which is a named tuple ``(args, kwargs)``. """ +from __future__ import absolute_import, division, print_function, with_statement +import collections import functools -import operator +import itertools import sys import types -class KeyReuseError(Exception): pass -class UnknownKeyError(Exception): pass -class LeakedCallbackError(Exception): pass -class BadYieldError(Exception): pass +from tornado.concurrent import Future, TracebackFuture +from tornado.ioloop import IOLoop +from tornado.stack_context import ExceptionStackContext, wrap + + +class KeyReuseError(Exception): + pass + + +class UnknownKeyError(Exception): + pass + + +class LeakedCallbackError(Exception): + pass + + +class BadYieldError(Exception): + pass + + +class ReturnValueIgnoredError(Exception): + pass + def engine(func): + """Callback-oriented decorator for asynchronous generators. + + This is an older interface; for new code that does not need to be + compatible with versions of Tornado older than 3.0 the + `coroutine` decorator is recommended instead. + + This decorator is similar to `coroutine`, except it does not + return a `.Future` and the ``callback`` argument is not treated + specially. + + In most cases, functions decorated with `engine` should take + a ``callback`` argument and invoke it with their result when + they are finished. One notable exception is the + `~tornado.web.RequestHandler` :ref:`HTTP verb methods `, + which use ``self.finish()`` in place of a callback argument. + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + runner = None + + def handle_exception(typ, value, tb): + # if the function throws an exception before its first "yield" + # (or is not a generator at all), the Runner won't exist yet. + # However, in that case we haven't reached anything asynchronous + # yet, so we can just let the exception propagate. + if runner is not None: + return runner.handle_exception(typ, value, tb) + return False + with ExceptionStackContext(handle_exception) as deactivate: + try: + result = func(*args, **kwargs) + except (Return, StopIteration) as e: + result = getattr(e, 'value', None) + else: + if isinstance(result, types.GeneratorType): + def final_callback(value): + if value is not None: + raise ReturnValueIgnoredError( + "@gen.engine functions cannot return values: " + "%r" % (value,)) + assert value is None + deactivate() + runner = Runner(result, final_callback) + runner.run() + return + if result is not None: + raise ReturnValueIgnoredError( + "@gen.engine functions cannot return values: %r" % + (result,)) + deactivate() + # no yield, so we're done + return wrapper + + +def coroutine(func): """Decorator for asynchronous generators. Any generator that yields objects from this module must be wrapped - in this decorator. The decorator only works on functions that are - already asynchronous. For `~tornado.web.RequestHandler` - ``get``/``post``/etc methods, this means that both the `tornado.gen.engine` - and `tornado.web.asynchronous` decorators must be used (in either order). - In most other cases, it means that it doesn't make sense to use - ``gen.engine`` on functions that don't already take a callback argument. + in either this decorator or `engine`. + + Coroutines may "return" by raising the special exception + `Return(value) `. In Python 3.3+, it is also possible for + the function to simply use the ``return value`` statement (prior to + Python 3.3 generators were not allowed to also return values). + In all versions of Python a coroutine that simply wishes to exit + early may use the ``return`` statement without a value. + + Functions with this decorator return a `.Future`. Additionally, + they may be called with a ``callback`` keyword argument, which + will be invoked with the future's result when it resolves. If the + coroutine fails, the callback will not be run and an exception + will be raised into the surrounding `.StackContext`. The + ``callback`` argument is not visible inside the decorated + function; it is handled by the decorator itself. + + From the caller's perspective, ``@gen.coroutine`` is similar to + the combination of ``@return_future`` and ``@gen.engine``. """ @functools.wraps(func) def wrapper(*args, **kwargs): - gen = func(*args, **kwargs) - if isinstance(gen, types.GeneratorType): - Runner(gen).run() - return - assert gen is None, gen - # no yield, so we're done + runner = None + future = TracebackFuture() + + if 'callback' in kwargs: + callback = kwargs.pop('callback') + IOLoop.current().add_future( + future, lambda future: callback(future.result())) + + def handle_exception(typ, value, tb): + try: + if runner is not None and runner.handle_exception(typ, value, tb): + return True + except Exception: + typ, value, tb = sys.exc_info() + future.set_exc_info((typ, value, tb)) + return True + with ExceptionStackContext(handle_exception) as deactivate: + try: + result = func(*args, **kwargs) + except (Return, StopIteration) as e: + result = getattr(e, 'value', None) + except Exception: + deactivate() + future.set_exc_info(sys.exc_info()) + return future + else: + if isinstance(result, types.GeneratorType): + def final_callback(value): + deactivate() + future.set_result(value) + runner = Runner(result, final_callback) + runner.run() + return future + deactivate() + future.set_result(result) + return future return wrapper + +class Return(Exception): + """Special exception to return a value from a `coroutine`. + + If this exception is raised, its value argument is used as the + result of the coroutine:: + + @gen.coroutine + def fetch_json(url): + response = yield AsyncHTTPClient().fetch(url) + raise gen.Return(json_decode(response.body)) + + In Python 3.3, this exception is no longer necessary: the ``return`` + statement can be used directly to return a value (previously + ``yield`` and ``return`` with a value could not be combined in the + same function). + + By analogy with the return statement, the value argument is optional, + but it is never necessary to ``raise gen.Return()``. The ``return`` + statement can be used with no arguments instead. + """ + def __init__(self, value=None): + super(Return, self).__init__() + self.value = value + + class YieldPoint(object): - """Base class for objects that may be yielded from the generator.""" + """Base class for objects that may be yielded from the generator. + + Applications do not normally need to use this class, but it may be + subclassed to provide additional yielding behavior. + """ def start(self, runner): """Called by the runner after the generator has yielded. - + No other methods will be called on this object before ``start``. """ raise NotImplementedError() @@ -112,12 +273,13 @@ def get_result(self): """Returns the value to use as the result of the yield expression. - + This method will only be called once, and only after `is_ready` has returned true. """ raise NotImplementedError() + class Callback(YieldPoint): """Returns a callable object that will allow a matching `Wait` to proceed. @@ -143,6 +305,7 @@ def get_result(self): return self.runner.result_callback(self.key) + class Wait(YieldPoint): """Returns the argument passed to the result of a previous `Callback`.""" def __init__(self, key): @@ -157,8 +320,9 @@ def get_result(self): return self.runner.pop_result(self.key) + class WaitAll(YieldPoint): - """Returns the results of multiple previous `Callbacks`. + """Returns the results of multiple previous `Callbacks `. The argument is a sequence of `Callback` keys, and the result is a list of results in the same order. @@ -173,10 +337,10 @@ def is_ready(self): return all(self.runner.is_ready(key) for key in self.keys) - + def get_result(self): return [self.runner.pop_result(key) for key in self.keys] - + class Task(YieldPoint): """Runs a single asynchronous operation. @@ -187,9 +351,9 @@ A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique key generated automatically):: - + result = yield gen.Task(func, args) - + func(args, callback=(yield gen.Callback(key))) result = yield gen.Wait(key) """ @@ -205,13 +369,32 @@ runner.register_callback(self.key) self.kwargs["callback"] = runner.result_callback(self.key) self.func(*self.args, **self.kwargs) - + def is_ready(self): return self.runner.is_ready(self.key) def get_result(self): return self.runner.pop_result(self.key) + +class YieldFuture(YieldPoint): + def __init__(self, future, io_loop=None): + self.future = future + self.io_loop = io_loop or IOLoop.current() + + def start(self, runner): + self.runner = runner + self.key = object() + runner.register_callback(self.key) + self.io_loop.add_future(self.future, runner.result_callback(self.key)) + + def is_ready(self): + return self.runner.is_ready(self.key) + + def get_result(self): + return self.runner.pop_result(self.key).result() + + class Multi(YieldPoint): """Runs multiple asynchronous operations in parallel. @@ -221,51 +404,70 @@ a list of ``YieldPoints``. """ def __init__(self, children): - assert all(isinstance(i, YieldPoint) for i in children) - self.children = children - + self.children = [] + for i in children: + if isinstance(i, Future): + i = YieldFuture(i) + self.children.append(i) + assert all(isinstance(i, YieldPoint) for i in self.children) + self.unfinished_children = set(self.children) + def start(self, runner): for i in self.children: i.start(runner) def is_ready(self): - return all(i.is_ready() for i in self.children) + finished = list(itertools.takewhile( + lambda i: i.is_ready(), self.unfinished_children)) + self.unfinished_children.difference_update(finished) + return not self.unfinished_children def get_result(self): return [i.get_result() for i in self.children] + class _NullYieldPoint(YieldPoint): def start(self, runner): pass + def is_ready(self): return True + def get_result(self): return None + +_null_yield_point = _NullYieldPoint() + + class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. + + ``final_callback`` is run after the generator exits. """ - def __init__(self, gen): + def __init__(self, gen, final_callback): self.gen = gen - self.yield_point = _NullYieldPoint() + self.final_callback = final_callback + self.yield_point = _null_yield_point self.pending_callbacks = set() self.results = {} self.running = False self.finished = False self.exc_info = None + self.had_exception = False def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if key in self.pending_callbacks: - raise KeyReuseError("key %r is already pending" % key) + raise KeyReuseError("key %r is already pending" % (key,)) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if key not in self.pending_callbacks: - raise UnknownKeyError("key %r is not pending" % key) + raise UnknownKeyError("key %r is not pending" % (key,)) return key in self.results def set_result(self, key, result): @@ -292,32 +494,48 @@ if not self.yield_point.is_ready(): return next = self.yield_point.get_result() + self.yield_point = None except Exception: self.exc_info = sys.exc_info() try: if self.exc_info is not None: + self.had_exception = True exc_info = self.exc_info self.exc_info = None yielded = self.gen.throw(*exc_info) else: yielded = self.gen.send(next) - except StopIteration: + except (StopIteration, Return) as e: self.finished = True - if self.pending_callbacks: + self.yield_point = _null_yield_point + if self.pending_callbacks and not self.had_exception: + # If we ran cleanly without waiting on all callbacks + # raise an error (really more of a warning). If we + # had an exception then some callbacks may have been + # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) + self.final_callback(getattr(e, 'value', None)) + self.final_callback = None return except Exception: self.finished = True + self.yield_point = _null_yield_point raise if isinstance(yielded, list): yielded = Multi(yielded) + elif isinstance(yielded, Future): + yielded = YieldFuture(yielded) if isinstance(yielded, YieldPoint): self.yield_point = yielded - self.yield_point.start(self) + try: + self.yield_point.start(self) + except Exception: + self.exc_info = sys.exc_info() else: - self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),) + self.exc_info = (BadYieldError( + "yielded unknown object %r" % (yielded,)),) finally: self.running = False @@ -330,20 +548,14 @@ else: result = None self.set_result(key, result) - return inner - -# in python 2.6+ this could be a collections.namedtuple -class Arguments(tuple): - """The result of a yield expression whose callback had more than one - argument (or keyword arguments). - - The `Arguments` object can be used as a tuple ``(args, kwargs)`` - or an object with attributes ``args`` and ``kwargs``. - """ - __slots__ = () + return wrap(inner) - def __new__(cls, args, kwargs): - return tuple.__new__(cls, (args, kwargs)) + def handle_exception(self, typ, value, tb): + if not self.running and not self.finished: + self.exc_info = (typ, value, tb) + self.run() + return True + else: + return False - args = property(operator.itemgetter(0)) - kwargs = property(operator.itemgetter(1)) +Arguments = collections.namedtuple('Arguments', ['args', 'kwargs']) diff -Nru python-tornado-2.1.0/tornado/httpclient.py python-tornado-3.1.1/tornado/httpclient.py --- python-tornado-2.1.0/tornado/httpclient.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/httpclient.py 2013-09-01 18:41:35.000000000 +0000 @@ -1,40 +1,44 @@ """Blocking and non-blocking HTTP client interfaces. This module defines a common interface shared by two implementations, -`simple_httpclient` and `curl_httpclient`. Applications may either +``simple_httpclient`` and ``curl_httpclient``. Applications may either instantiate their chosen implementation class directly or use the `AsyncHTTPClient` class from this module, which selects an implementation that can be overridden with the `AsyncHTTPClient.configure` method. -The default implementation is `simple_httpclient`, and this is expected +The default implementation is ``simple_httpclient``, and this is expected to be suitable for most users' needs. However, some applications may wish -to switch to `curl_httpclient` for reasons such as the following: +to switch to ``curl_httpclient`` for reasons such as the following: -* `curl_httpclient` is more likely to be compatible with sites that are +* ``curl_httpclient`` has some features not found in ``simple_httpclient``, + including support for HTTP proxies and the ability to use a specified + network interface. + +* ``curl_httpclient`` is more likely to be compatible with sites that are not-quite-compliant with the HTTP spec, or sites that use little-exercised features of HTTP. -* `simple_httpclient` only supports SSL on Python 2.6 and above. - -* `curl_httpclient` is faster +* ``curl_httpclient`` is faster. -* `curl_httpclient` was the default prior to Tornado 2.0. +* ``curl_httpclient`` was the default prior to Tornado 2.0. -Note that if you are using `curl_httpclient`, it is highly recommended that +Note that if you are using ``curl_httpclient``, it is highly recommended that you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum supported version is 7.18.2, and the recommended version is 7.21.1 or newer. """ -import calendar -import email.utils -import httplib +from __future__ import absolute_import, division, print_function, with_statement + +import functools import time import weakref +from tornado.concurrent import Future from tornado.escape import utf8 -from tornado import httputil +from tornado import httputil, stack_context from tornado.ioloop import IOLoop -from tornado.util import import_object, bytes_type +from tornado.util import Configurable + class HTTPClient(object): """A blocking HTTP client. @@ -47,13 +51,15 @@ try: response = http_client.fetch("http://www.google.com/") print response.body - except httpclient.HTTPError, e: + except httpclient.HTTPError as e: print "Error:", e + http_client.close() """ - def __init__(self): + def __init__(self, async_client_class=None, **kwargs): self._io_loop = IOLoop() - self._async_client = AsyncHTTPClient(self._io_loop) - self._response = None + if async_client_class is None: + async_client_class = AsyncHTTPClient + self._async_client = async_client_class(self._io_loop, **kwargs) self._closed = False def __del__(self): @@ -68,203 +74,252 @@ def fetch(self, request, **kwargs): """Executes a request, returning an `HTTPResponse`. - + The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` If an error occurs during the fetch, we raise an `HTTPError`. """ - def callback(response): - self._response = response - self._io_loop.stop() - self._async_client.fetch(request, callback, **kwargs) - self._io_loop.start() - response = self._response - self._response = None + response = self._io_loop.run_sync(functools.partial( + self._async_client.fetch, request, **kwargs)) response.rethrow() return response -class AsyncHTTPClient(object): + +class AsyncHTTPClient(Configurable): """An non-blocking HTTP client. Example usage:: - import ioloop - def handle_request(response): if response.error: print "Error:", response.error else: print response.body - ioloop.IOLoop.instance().stop() - http_client = httpclient.AsyncHTTPClient() + http_client = AsyncHTTPClient() http_client.fetch("http://www.google.com/", handle_request) - ioloop.IOLoop.instance().start() - The constructor for this class is magic in several respects: It actually - creates an instance of an implementation-specific subclass, and instances - are reused as a kind of pseudo-singleton (one per IOLoop). The keyword - argument force_instance=True can be used to suppress this singleton - behavior. Constructor arguments other than io_loop and force_instance - are deprecated. The implementation subclass as well as arguments to - its constructor can be set with the static method configure() + The constructor for this class is magic in several respects: It + actually creates an instance of an implementation-specific + subclass, and instances are reused as a kind of pseudo-singleton + (one per `.IOLoop`). The keyword argument ``force_instance=True`` + can be used to suppress this singleton behavior. Constructor + arguments other than ``io_loop`` and ``force_instance`` are + deprecated. The implementation subclass as well as arguments to + its constructor can be set with the static method `configure()` """ - _impl_class = None - _impl_kwargs = None + @classmethod + def configurable_base(cls): + return AsyncHTTPClient + + @classmethod + def configurable_default(cls): + from tornado.simple_httpclient import SimpleAsyncHTTPClient + return SimpleAsyncHTTPClient @classmethod def _async_clients(cls): - assert cls is not AsyncHTTPClient, "should only be called on subclasses" - if not hasattr(cls, '_async_client_dict'): - cls._async_client_dict = weakref.WeakKeyDictionary() - return cls._async_client_dict - - def __new__(cls, io_loop=None, max_clients=10, force_instance=False, - **kwargs): - io_loop = io_loop or IOLoop.instance() - if cls is AsyncHTTPClient: - if cls._impl_class is None: - from tornado.simple_httpclient import SimpleAsyncHTTPClient - AsyncHTTPClient._impl_class = SimpleAsyncHTTPClient - impl = AsyncHTTPClient._impl_class - else: - impl = cls - if io_loop in impl._async_clients() and not force_instance: - return impl._async_clients()[io_loop] - else: - instance = super(AsyncHTTPClient, cls).__new__(impl) - args = {} - if cls._impl_kwargs: - args.update(cls._impl_kwargs) - args.update(kwargs) - instance.initialize(io_loop, max_clients, **args) - if not force_instance: - impl._async_clients()[io_loop] = instance - return instance + attr_name = '_async_client_dict_' + cls.__name__ + if not hasattr(cls, attr_name): + setattr(cls, attr_name, weakref.WeakKeyDictionary()) + return getattr(cls, attr_name) + + def __new__(cls, io_loop=None, force_instance=False, **kwargs): + io_loop = io_loop or IOLoop.current() + if io_loop in cls._async_clients() and not force_instance: + return cls._async_clients()[io_loop] + instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, + **kwargs) + if not force_instance: + cls._async_clients()[io_loop] = instance + return instance + + def initialize(self, io_loop, defaults=None): + self.io_loop = io_loop + self.defaults = dict(HTTPRequest._DEFAULTS) + if defaults is not None: + self.defaults.update(defaults) def close(self): - """Destroys this http client, freeing any file descriptors used. + """Destroys this HTTP client, freeing any file descriptors used. Not needed in normal use, but may be helpful in unittests that create and destroy http clients. No other methods may be called - on the AsyncHTTPClient after close(). + on the `AsyncHTTPClient` after ``close()``. """ if self._async_clients().get(self.io_loop) is self: del self._async_clients()[self.io_loop] - def fetch(self, request, callback, **kwargs): - """Executes a request, calling callback with an `HTTPResponse`. + def fetch(self, request, callback=None, **kwargs): + """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` - If an error occurs during the fetch, the HTTPResponse given to the - callback has a non-None error attribute that contains the exception - encountered during the request. You can call response.rethrow() to - throw the exception (if any) in the callback. + This method returns a `.Future` whose result is an + `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if + the request returned a non-200 response code. + + If a ``callback`` is given, it will be invoked with the `HTTPResponse`. + In the callback interface, `HTTPError` is not automatically raised. + Instead, you must check the response's ``error`` attribute or + call its `~HTTPResponse.rethrow` method. """ + if not isinstance(request, HTTPRequest): + request = HTTPRequest(url=request, **kwargs) + # We may modify this (to add Host, Accept-Encoding, etc), + # so make sure we don't modify the caller's object. This is also + # where normal dicts get converted to HTTPHeaders objects. + request.headers = httputil.HTTPHeaders(request.headers) + request = _RequestProxy(request, self.defaults) + future = Future() + if callback is not None: + callback = stack_context.wrap(callback) + + def handle_future(future): + exc = future.exception() + if isinstance(exc, HTTPError) and exc.response is not None: + response = exc.response + elif exc is not None: + response = HTTPResponse( + request, 599, error=exc, + request_time=time.time() - request.start_time) + else: + response = future.result() + self.io_loop.add_callback(callback, response) + future.add_done_callback(handle_future) + + def handle_response(response): + if response.error: + future.set_exception(response.error) + else: + future.set_result(response) + self.fetch_impl(request, handle_response) + return future + + def fetch_impl(self, request, callback): raise NotImplementedError() - @staticmethod - def configure(impl, **kwargs): - """Configures the AsyncHTTPClient subclass to use. + @classmethod + def configure(cls, impl, **kwargs): + """Configures the `AsyncHTTPClient` subclass to use. - AsyncHTTPClient() actually creates an instance of a subclass. + ``AsyncHTTPClient()`` actually creates an instance of a subclass. This method may be called with either a class object or the - fully-qualified name of such a class (or None to use the default, - SimpleAsyncHTTPClient) + fully-qualified name of such a class (or ``None`` to use the default, + ``SimpleAsyncHTTPClient``) If additional keyword arguments are given, they will be passed to the constructor of each subclass instance created. The - keyword argument max_clients determines the maximum number of - simultaneous fetch() operations that can execute in parallel - on each IOLoop. Additional arguments may be supported depending - on the implementation class in use. + keyword argument ``max_clients`` determines the maximum number + of simultaneous `~AsyncHTTPClient.fetch()` operations that can + execute in parallel on each `.IOLoop`. Additional arguments + may be supported depending on the implementation class in use. Example:: AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") """ - if isinstance(impl, (unicode, bytes_type)): - impl = import_object(impl) - if impl is not None and not issubclass(impl, AsyncHTTPClient): - raise ValueError("Invalid AsyncHTTPClient implementation") - AsyncHTTPClient._impl_class = impl - AsyncHTTPClient._impl_kwargs = kwargs + super(AsyncHTTPClient, cls).configure(impl, **kwargs) + class HTTPRequest(object): """HTTP client request object.""" + + # Default values for HTTPRequest parameters. + # Merged with the values on the request object by AsyncHTTPClient + # implementations. + _DEFAULTS = dict( + connect_timeout=20.0, + request_timeout=20.0, + follow_redirects=True, + max_redirects=5, + use_gzip=True, + proxy_password='', + allow_nonstandard_methods=False, + validate_cert=True) + def __init__(self, url, method="GET", headers=None, body=None, - auth_username=None, auth_password=None, - connect_timeout=20.0, request_timeout=20.0, - if_modified_since=None, follow_redirects=True, - max_redirects=5, user_agent=None, use_gzip=True, + auth_username=None, auth_password=None, auth_mode=None, + connect_timeout=None, request_timeout=None, + if_modified_since=None, follow_redirects=None, + max_redirects=None, user_agent=None, use_gzip=None, network_interface=None, streaming_callback=None, header_callback=None, prepare_curl_callback=None, proxy_host=None, proxy_port=None, proxy_username=None, - proxy_password='', allow_nonstandard_methods=False, - validate_cert=True, ca_certs=None, + proxy_password=None, allow_nonstandard_methods=None, + validate_cert=None, ca_certs=None, allow_ipv6=None, client_key=None, client_cert=None): - """Creates an `HTTPRequest`. - - All parameters except `url` are optional. + r"""All parameters except ``url`` are optional. :arg string url: URL to fetch :arg string method: HTTP method, e.g. "GET" or "POST" :arg headers: Additional HTTP headers to pass on the request + :arg body: HTTP body to pass on the request :type headers: `~tornado.httputil.HTTPHeaders` or `dict` - :arg string auth_username: Username for HTTP "Basic" authentication - :arg string auth_password: Password for HTTP "Basic" authentication + :arg string auth_username: Username for HTTP authentication + :arg string auth_password: Password for HTTP authentication + :arg string auth_mode: Authentication mode; default is "basic". + Allowed values are implementation-defined; ``curl_httpclient`` + supports "basic" and "digest"; ``simple_httpclient`` only supports + "basic" :arg float connect_timeout: Timeout for initial connection in seconds :arg float request_timeout: Timeout for entire request in seconds - :arg datetime if_modified_since: Timestamp for ``If-Modified-Since`` - header + :arg if_modified_since: Timestamp for ``If-Modified-Since`` header + :type if_modified_since: `datetime` or `float` :arg bool follow_redirects: Should redirects be followed automatically or return the 3xx response? - :arg int max_redirects: Limit for `follow_redirects` + :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server :arg string network_interface: Network interface to use for request - :arg callable streaming_callback: If set, `streaming_callback` will - be run with each chunk of data as it is received, and - `~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in + :arg callable streaming_callback: If set, ``streaming_callback`` will + be run with each chunk of data as it is received, and + ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in the final response. - :arg callable header_callback: If set, `header_callback` will - be run with each header line as it is received, and - `~HTTPResponse.headers` will be empty in the final response. + :arg callable header_callback: If set, ``header_callback`` will + be run with each header line as it is received (including the + first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line + containing only ``\r\n``. All lines include the trailing newline + characters). ``HTTPResponse.headers`` will be empty in the final + response. This is most useful in conjunction with + ``streaming_callback``, because it's the only way to get access to + header data while the request is in progress. :arg callable prepare_curl_callback: If set, will be called with - a `pycurl.Curl` object to allow the application to make additional - `setopt` calls. - :arg string proxy_host: HTTP proxy hostname. To use proxies, - `proxy_host` and `proxy_port` must be set; `proxy_username` and - `proxy_pass` are optional. Proxies are currently only support - with `curl_httpclient`. + a ``pycurl.Curl`` object to allow the application to make additional + ``setopt`` calls. + :arg string proxy_host: HTTP proxy hostname. To use proxies, + ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and + ``proxy_pass`` are optional. Proxies are currently only supported + with ``curl_httpclient``. :arg int proxy_port: HTTP proxy port :arg string proxy_username: HTTP proxy username :arg string proxy_password: HTTP proxy password - :arg bool allow_nonstandard_methods: Allow unknown values for `method` + :arg bool allow_nonstandard_methods: Allow unknown values for ``method`` argument? :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. Note that in `curl_httpclient`, if - any request uses a custom `ca_certs` file, they all must (they - don't have to all use the same `ca_certs`, but it's not possible - to mix requests with ca_certs and requests that use the defaults. - :arg bool allow_ipv6: Use IPv6 when available? Default is false in - `simple_httpclient` and true in `curl_httpclient` + or None to use defaults. Note that in ``curl_httpclient``, if + any request uses a custom ``ca_certs`` file, they all must (they + don't have to all use the same ``ca_certs``, but it's not possible + to mix requests with ``ca_certs`` and requests that use the defaults. + :arg bool allow_ipv6: Use IPv6 when available? Default is false in + ``simple_httpclient`` and true in ``curl_httpclient`` :arg string client_key: Filename for client SSL key, if any :arg string client_cert: Filename for client SSL certificate, if any + + .. versionadded:: 3.1 + The ``auth_mode`` argument. """ if headers is None: headers = httputil.HTTPHeaders() if if_modified_since: - timestamp = calendar.timegm(if_modified_since.utctimetuple()) - headers["If-Modified-Since"] = email.utils.formatdate( - timestamp, localtime=False, usegmt=True) + headers["If-Modified-Since"] = httputil.format_timestamp( + if_modified_since) self.proxy_host = proxy_host self.proxy_port = proxy_port self.proxy_username = proxy_username @@ -275,6 +330,7 @@ self.body = utf8(body) self.auth_username = auth_username self.auth_password = auth_password + self.auth_mode = auth_mode self.connect_timeout = connect_timeout self.request_timeout = request_timeout self.follow_redirects = follow_redirects @@ -282,9 +338,9 @@ self.user_agent = user_agent self.use_gzip = use_gzip self.network_interface = network_interface - self.streaming_callback = streaming_callback - self.header_callback = header_callback - self.prepare_curl_callback = prepare_curl_callback + self.streaming_callback = stack_context.wrap(streaming_callback) + self.header_callback = stack_context.wrap(header_callback) + self.prepare_curl_callback = stack_context.wrap(prepare_curl_callback) self.allow_nonstandard_methods = allow_nonstandard_methods self.validate_cert = validate_cert self.ca_certs = ca_certs @@ -303,28 +359,39 @@ * code: numeric HTTP status code, e.g. 200 or 404 - * headers: httputil.HTTPHeaders object + * reason: human-readable reason phrase describing the status code + (with curl_httpclient, this is a default value rather than the + server's actual response) + + * headers: `tornado.httputil.HTTPHeaders` object - * buffer: cStringIO object for response body + * buffer: ``cStringIO`` object for response body - * body: respose body as string (created on demand from self.buffer) + * body: response body as string (created on demand from ``self.buffer``) * error: Exception object, if any * request_time: seconds from request start to finish * time_info: dictionary of diagnostic timing information from the request. - Available data are subject to change, but currently uses timings - available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, - plus 'queue', which is the delay (if any) introduced by waiting for - a slot under AsyncHTTPClient's max_clients setting. + Available data are subject to change, but currently uses timings + available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, + plus ``queue``, which is the delay (if any) introduced by waiting for + a slot under `AsyncHTTPClient`'s ``max_clients`` setting. """ - def __init__(self, request, code, headers={}, buffer=None, + def __init__(self, request, code, headers=None, buffer=None, effective_url=None, error=None, request_time=None, - time_info={}): - self.request = request + time_info=None, reason=None): + if isinstance(request, _RequestProxy): + self.request = request.request + else: + self.request = request self.code = code - self.headers = headers + self.reason = reason or httputil.responses.get(code, "Unknown") + if headers is not None: + self.headers = headers + else: + self.headers = httputil.HTTPHeaders() self.buffer = buffer self._body = None if effective_url is None: @@ -339,7 +406,7 @@ else: self.error = error self.request_time = request_time - self.time_info = time_info + self.time_info = time_info or {} def _get_body(self): if self.buffer is None: @@ -357,7 +424,7 @@ raise self.error def __repr__(self): - args = ",".join("%s=%r" % i for i in self.__dict__.iteritems()) + args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items())) return "%s(%s)" % (self.__class__.__name__, args) @@ -366,42 +433,64 @@ Attributes: - code - HTTP error integer error code, e.g. 404. Error code 599 is - used when no HTTP response was received, e.g. for a timeout. + * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is + used when no HTTP response was received, e.g. for a timeout. - response - HTTPResponse object, if any. + * ``response`` - `HTTPResponse` object, if any. - Note that if follow_redirects is False, redirects become HTTPErrors, - and you can look at error.response.headers['Location'] to see the + Note that if ``follow_redirects`` is False, redirects become HTTPErrors, + and you can look at ``error.response.headers['Location']`` to see the destination of the redirect. """ def __init__(self, code, message=None, response=None): self.code = code - message = message or httplib.responses.get(code, "Unknown") + message = message or httputil.responses.get(code, "Unknown") self.response = response Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) +class _RequestProxy(object): + """Combines an object with a dictionary of defaults. + + Used internally by AsyncHTTPClient implementations. + """ + def __init__(self, request, defaults): + self.request = request + self.defaults = defaults + + def __getattr__(self, name): + request_attr = getattr(self.request, name) + if request_attr is not None: + return request_attr + elif self.defaults is not None: + return self.defaults.get(name, None) + else: + return None + + def main(): from tornado.options import define, options, parse_command_line define("print_headers", type=bool, default=False) define("print_body", type=bool, default=True) define("follow_redirects", type=bool, default=True) + define("validate_cert", type=bool, default=True) args = parse_command_line() client = HTTPClient() for arg in args: try: response = client.fetch(arg, - follow_redirects=options.follow_redirects) - except HTTPError, e: + follow_redirects=options.follow_redirects, + validate_cert=options.validate_cert, + ) + except HTTPError as e: if e.response is not None: response = e.response else: raise if options.print_headers: - print response.headers + print(response.headers) if options.print_body: - print response.body + print(response.body) client.close() if __name__ == "__main__": diff -Nru python-tornado-2.1.0/tornado/httpserver.py python-tornado-3.1.1/tornado/httpserver.py --- python-tornado-2.1.0/tornado/httpserver.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/httpserver.py 2013-08-04 19:34:21.000000000 +0000 @@ -24,23 +24,26 @@ `tornado.web.RequestHandler.request`. """ -import Cookie -import logging +from __future__ import absolute_import, division, print_function, with_statement + import socket +import ssl import time -import urlparse -from tornado.escape import utf8, native_str, parse_qs_bytes +from tornado.escape import native_str, parse_qs_bytes from tornado import httputil from tornado import iostream -from tornado.netutil import TCPServer +from tornado.log import gen_log +from tornado import netutil +from tornado.tcpserver import TCPServer from tornado import stack_context -from tornado.util import b, bytes_type +from tornado.util import bytes_type try: - import ssl # Python 2.6+ + import Cookie # py2 except ImportError: - ssl = None + import http.cookies as Cookie # py3 + class HTTPServer(TCPServer): r"""A non-blocking, single-threaded HTTP server. @@ -52,8 +55,8 @@ requests). A simple example server that echoes back the URI you requested:: - import httpserver - import ioloop + import tornado.httpserver + import tornado.ioloop def handle_request(request): message = "You requested %s\n" % request.uri @@ -61,30 +64,39 @@ len(message), message)) request.finish() - http_server = httpserver.HTTPServer(handle_request) + http_server = tornado.httpserver.HTTPServer(handle_request) http_server.listen(8888) - ioloop.IOLoop.instance().start() + tornado.ioloop.IOLoop.instance().start() - `HTTPServer` is a very basic connection handler. Beyond parsing the - HTTP request body and headers, the only HTTP semantics implemented - in `HTTPServer` is HTTP/1.1 keep-alive connections. We do not, however, - implement chunked encoding, so the request callback must provide a - ``Content-Length`` header or implement chunked encoding for HTTP/1.1 - requests for the server to run correctly for HTTP/1.1 clients. If - the request handler is unable to do this, you can provide the - ``no_keep_alive`` argument to the `HTTPServer` constructor, which will - ensure the connection is closed on every request no matter what HTTP - version the client is using. - - If ``xheaders`` is ``True``, we support the ``X-Real-Ip`` and ``X-Scheme`` - headers, which override the remote IP and HTTP scheme for all requests. - These headers are useful when running Tornado behind a reverse proxy or - load balancer. + `HTTPServer` is a very basic connection handler. It parses the request + headers and body, but the request callback is responsible for producing + the response exactly as it will appear on the wire. This affords + maximum flexibility for applications to implement whatever parts + of HTTP responses are required. + + `HTTPServer` supports keep-alive connections by default + (automatically for HTTP/1.1, or for HTTP/1.0 when the client + requests ``Connection: keep-alive``). This means that the request + callback must generate a properly-framed response, using either + the ``Content-Length`` header or ``Transfer-Encoding: chunked``. + Applications that are unable to frame their responses properly + should instead return a ``Connection: close`` header in each + response and pass ``no_keep_alive=True`` to the `HTTPServer` + constructor. + + If ``xheaders`` is ``True``, we support the + ``X-Real-Ip``/``X-Forwarded-For`` and + ``X-Scheme``/``X-Forwarded-Proto`` headers, which override the + remote IP and URI scheme/protocol for all requests. These headers + are useful when running Tornado behind a reverse proxy or load + balancer. The ``protocol`` argument can also be set to ``https`` + if Tornado is run behind an SSL-decoding proxy that does not set one of + the supported ``xheaders``. - `HTTPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. - To make this server serve SSL traffic, send the ssl_options dictionary + To make this server serve SSL traffic, send the ``ssl_options`` dictionary argument with the arguments required for the `ssl.wrap_socket` method, - including "certfile" and "keyfile":: + including ``certfile`` and ``keyfile``. (In Python 3.2+ you can pass + an `ssl.SSLContext` object instead of a dict):: HTTPServer(applicaton, ssl_options={ "certfile": os.path.join(data_dir, "mydomain.crt"), @@ -92,9 +104,9 @@ }) `HTTPServer` initialization follows one of three patterns (the - initialization methods are defined on `tornado.netutil.TCPServer`): + initialization methods are defined on `tornado.tcpserver.TCPServer`): - 1. `~tornado.netutil.TCPServer.listen`: simple single-process:: + 1. `~tornado.tcpserver.TCPServer.listen`: simple single-process:: server = HTTPServer(app) server.listen(8888) @@ -103,7 +115,7 @@ In many cases, `tornado.web.Application.listen` can be used to avoid the need to explicitly create the `HTTPServer`. - 2. `~tornado.netutil.TCPServer.bind`/`~tornado.netutil.TCPServer.start`: + 2. `~tornado.tcpserver.TCPServer.bind`/`~tornado.tcpserver.TCPServer.start`: simple multi-process:: server = HTTPServer(app) @@ -111,11 +123,11 @@ server.start(0) # Forks multiple sub-processes IOLoop.instance().start() - When using this interface, an `IOLoop` must *not* be passed - to the `HTTPServer` constructor. `start` will always start - the server on the default singleton `IOLoop`. + When using this interface, an `.IOLoop` must *not* be passed + to the `HTTPServer` constructor. `~.TCPServer.start` will always start + the server on the default singleton `.IOLoop`. - 3. `~tornado.netutil.TCPServer.add_sockets`: advanced multi-process:: + 3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process:: sockets = tornado.netutil.bind_sockets(8888) tornado.process.fork_processes(0) @@ -123,30 +135,33 @@ server.add_sockets(sockets) IOLoop.instance().start() - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `tornado.netutil.bind_sockets`. + The `~.TCPServer.add_sockets` interface is more complicated, + but it can be used with `tornado.process.fork_processes` to + give you more flexibility in when the fork happens. + `~.TCPServer.add_sockets` can also be used in single-process + servers if you want to create your listening sockets in some + way other than `tornado.netutil.bind_sockets`. """ def __init__(self, request_callback, no_keep_alive=False, io_loop=None, - xheaders=False, ssl_options=None, **kwargs): + xheaders=False, ssl_options=None, protocol=None, **kwargs): self.request_callback = request_callback self.no_keep_alive = no_keep_alive self.xheaders = xheaders + self.protocol = protocol TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, **kwargs) def handle_stream(self, stream, address): HTTPConnection(stream, address, self.request_callback, - self.no_keep_alive, self.xheaders) + self.no_keep_alive, self.xheaders, self.protocol) + class _BadRequestException(Exception): """Exception class for malformed HTTP requests.""" pass + class HTTPConnection(object): """Handles a connection to an HTTP client, executing HTTP requests. @@ -154,34 +169,75 @@ until the HTTP conection is closed. """ def __init__(self, stream, address, request_callback, no_keep_alive=False, - xheaders=False): + xheaders=False, protocol=None): self.stream = stream - if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6): - # Unix (or other) socket; fake the remote address - address = ('0.0.0.0', 0) self.address = address + # Save the socket's address family now so we know how to + # interpret self.address even after the stream is closed + # and its socket attribute replaced with None. + self.address_family = stream.socket.family self.request_callback = request_callback self.no_keep_alive = no_keep_alive self.xheaders = xheaders - self._request = None - self._request_finished = False + self.protocol = protocol + self._clear_request_state() # Save stack context here, outside of any request. This keeps # contexts from one request from leaking into the next. self._header_callback = stack_context.wrap(self._on_headers) - self.stream.read_until(b("\r\n\r\n"), self._header_callback) + self.stream.set_close_callback(self._on_connection_close) + self.stream.read_until(b"\r\n\r\n", self._header_callback) + + def _clear_request_state(self): + """Clears the per-request state. + + This is run in between requests to allow the previous handler + to be garbage collected (and prevent spurious close callbacks), + and when the connection is closed (to break up cycles and + facilitate garbage collection in cpython). + """ + self._request = None + self._request_finished = False self._write_callback = None + self._close_callback = None + + def set_close_callback(self, callback): + """Sets a callback that will be run when the connection is closed. + + Use this instead of accessing + `HTTPConnection.stream.set_close_callback + <.BaseIOStream.set_close_callback>` directly (which was the + recommended approach prior to Tornado 3.0). + """ + self._close_callback = stack_context.wrap(callback) + + def _on_connection_close(self): + if self._close_callback is not None: + callback = self._close_callback + self._close_callback = None + callback() + # Delete any unfinished callbacks to break up reference cycles. + self._header_callback = None + self._clear_request_state() + + def close(self): + self.stream.close() + # Remove this reference to self, which would otherwise cause a + # cycle and delay garbage collection of this connection. + self._header_callback = None + self._clear_request_state() def write(self, chunk, callback=None): """Writes a chunk of output to the stream.""" - assert self._request, "Request closed" if not self.stream.closed(): self._write_callback = stack_context.wrap(callback) self.stream.write(chunk, self._on_write_complete) def finish(self): """Finishes the request.""" - assert self._request, "Request closed" self._request_finished = True + # No more data is coming, so instruct TCP to send any remaining + # data immediately instead of waiting for a full packet or ack. + self.stream.set_nodelay(True) if not self.stream.writing(): self._finish_request() @@ -189,28 +245,46 @@ if self._write_callback is not None: callback = self._write_callback self._write_callback = None - callback() - if self._request_finished: + callback() + # _on_write_complete is enqueued on the IOLoop whenever the + # IOStream's write buffer becomes empty, but it's possible for + # another callback that runs on the IOLoop before it to + # simultaneously write more data and finish the request. If + # there is still data in the IOStream, a future + # _on_write_complete will be responsible for calling + # _finish_request. + if self._request_finished and not self.stream.writing(): self._finish_request() def _finish_request(self): - if self.no_keep_alive: + if self.no_keep_alive or self._request is None: disconnect = True else: connection_header = self._request.headers.get("Connection") + if connection_header is not None: + connection_header = connection_header.lower() if self._request.supports_http_1_1(): disconnect = connection_header == "close" elif ("Content-Length" in self._request.headers or self._request.method in ("HEAD", "GET")): - disconnect = connection_header != "Keep-Alive" + disconnect = connection_header != "keep-alive" else: disconnect = True - self._request = None - self._request_finished = False + self._clear_request_state() if disconnect: - self.stream.close() + self.close() return - self.stream.read_until(b("\r\n\r\n"), self._header_callback) + try: + # Use a try/except instead of checking stream.closed() + # directly, because in some cases the stream doesn't discover + # that it's closed until you try to read from it. + self.stream.read_until(b"\r\n\r\n", self._header_callback) + + # Turn Nagle's algorithm back on, leaving the stream in its + # default state for the next request. + self.stream.set_nodelay(False) + except iostream.StreamClosedError: + self.close() def _on_headers(self, data): try: @@ -223,10 +297,22 @@ raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException("Malformed HTTP version in HTTP Request-Line") - headers = httputil.HTTPHeaders.parse(data[eol:]) + try: + headers = httputil.HTTPHeaders.parse(data[eol:]) + except ValueError: + # Probably from split() if there was no ':' in the line + raise _BadRequestException("Malformed HTTP headers") + + # HTTPRequest wants an IP, not a full socket address + if self.address_family in (socket.AF_INET, socket.AF_INET6): + remote_ip = self.address[0] + else: + # Unix (or other) socket; fake the remote address + remote_ip = '0.0.0.0' + self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, - headers=headers, remote_ip=self.address[0]) + headers=headers, remote_ip=remote_ip, protocol=self.protocol) content_length = headers.get("Content-Length") if content_length: @@ -234,40 +320,23 @@ if content_length > self.stream.max_buffer_size: raise _BadRequestException("Content-Length too long") if headers.get("Expect") == "100-continue": - self.stream.write(b("HTTP/1.1 100 (Continue)\r\n\r\n")) + self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") self.stream.read_bytes(content_length, self._on_request_body) return self.request_callback(self._request) - except _BadRequestException, e: - logging.info("Malformed HTTP request from %s: %s", + except _BadRequestException as e: + gen_log.info("Malformed HTTP request from %s: %s", self.address[0], e) - self.stream.close() + self.close() return def _on_request_body(self, data): self._request.body = data - content_type = self._request.headers.get("Content-Type", "") - if self._request.method in ("POST", "PUT"): - if content_type.startswith("application/x-www-form-urlencoded"): - arguments = parse_qs_bytes(native_str(self._request.body)) - for name, values in arguments.iteritems(): - values = [v for v in values if v] - if values: - self._request.arguments.setdefault(name, []).extend( - values) - elif content_type.startswith("multipart/form-data"): - fields = content_type.split(";") - for field in fields: - k, sep, v = field.strip().partition("=") - if k == "boundary" and v: - httputil.parse_multipart_form_data( - utf8(v), data, - self._request.arguments, - self._request.files) - break - else: - logging.warning("Invalid multipart/form-data") + if self._request.method in ("POST", "PATCH", "PUT"): + httputil.parse_body_arguments( + self._request.headers.get("Content-Type", ""), data, + self._request.arguments, self._request.files) self.request_callback(self._request) @@ -298,7 +367,7 @@ .. attribute:: headers - `HTTPHeader` dictionary-like object for request headers. Acts like + `.HTTPHeaders` dictionary-like object for request headers. Acts like a case-insensitive dictionary with additional methods for repeated headers. @@ -308,14 +377,17 @@ .. attribute:: remote_ip - Client's IP address as a string. If `HTTPServer.xheaders` is set, + Client's IP address as a string. If ``HTTPServer.xheaders`` is set, will pass along the real IP address provided by a load balancer - in the ``X-Real-Ip`` header + in the ``X-Real-Ip`` or ``X-Forwarded-For`` header. + + .. versionchanged:: 3.1 + The list format of ``X-Forwarded-For`` is now supported. .. attribute:: protocol - The protocol used, either "http" or "https". If `HTTPServer.xheaders` - is seet, will pass along the protocol used by a load balancer if + The protocol used, either "http" or "https". If ``HTTPServer.xheaders`` + is set, will pass along the protocol used by a load balancer if reported via an ``X-Scheme`` header. .. attribute:: host @@ -327,14 +399,14 @@ GET/POST arguments are available in the arguments property, which maps arguments names to lists of values (to support multiple values for individual names). Names are of type `str`, while arguments - are byte strings. Note that this is different from - `RequestHandler.get_argument`, which returns argument values as + are byte strings. Note that this is different from + `.RequestHandler.get_argument`, which returns argument values as unicode strings. .. attribute:: files File uploads are available in the files property, which maps file - names to lists of :class:`HTTPFile`. + names to lists of `.HTTPFile`. .. attribute:: connection @@ -351,38 +423,40 @@ self.version = version self.headers = headers or httputil.HTTPHeaders() self.body = body or "" + + # set remote IP and protocol + self.remote_ip = remote_ip + if protocol: + self.protocol = protocol + elif connection and isinstance(connection.stream, + iostream.SSLIOStream): + self.protocol = "https" + else: + self.protocol = "http" + + # xheaders can override the defaults if connection and connection.xheaders: # Squid uses X-Forwarded-For, others use X-Real-Ip - self.remote_ip = self.headers.get( - "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip)) + ip = self.headers.get("X-Forwarded-For", self.remote_ip) + ip = ip.split(',')[-1].strip() + ip = self.headers.get( + "X-Real-Ip", ip) + if netutil.is_valid_ip(ip): + self.remote_ip = ip # AWS uses X-Forwarded-Proto - self.protocol = self.headers.get( - "X-Scheme", self.headers.get("X-Forwarded-Proto", protocol)) - if self.protocol not in ("http", "https"): - self.protocol = "http" - else: - self.remote_ip = remote_ip - if protocol: - self.protocol = protocol - elif connection and isinstance(connection.stream, - iostream.SSLIOStream): - self.protocol = "https" - else: - self.protocol = "http" + proto = self.headers.get( + "X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol)) + if proto in ("http", "https"): + self.protocol = proto + self.host = host or self.headers.get("Host") or "127.0.0.1" self.files = files or {} self.connection = connection self._start_time = time.time() self._finish_time = None - scheme, netloc, path, query, fragment = urlparse.urlsplit(native_str(uri)) - self.path = path - self.query = query - arguments = parse_qs_bytes(query) - self.arguments = {} - for name, values in arguments.iteritems(): - values = [v for v in values if v] - if values: self.arguments[name] = values + self.path, sep, self.query = uri.partition('?') + self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics""" @@ -398,7 +472,7 @@ self._cookies.load( native_str(self.headers["Cookie"])) except Exception: - self._cookies = None + self._cookies = {} return self._cookies def write(self, chunk, callback=None): @@ -422,7 +496,7 @@ else: return self._finish_time - self._start_time - def get_ssl_certificate(self): + def get_ssl_certificate(self, binary_form=False): """Returns the client's SSL certificate, if any. To use client certificates, the HTTPServer must have been constructed @@ -435,18 +509,21 @@ cert_reqs=ssl.CERT_REQUIRED, ca_certs="cacert.crt")) - The return value is a dictionary, see SSLSocket.getpeercert() in - the standard library for more details. + By default, the return value is a dictionary (or None, if no + client certificate is present). If ``binary_form`` is true, a + DER-encoded form of the certificate is returned instead. See + SSLSocket.getpeercert() in the standard library for more + details. http://docs.python.org/library/ssl.html#sslsocket-objects """ try: - return self.connection.stream.socket.getpeercert() + return self.connection.stream.socket.getpeercert( + binary_form=binary_form) except ssl.SSLError: return None def __repr__(self): - attrs = ("protocol", "host", "method", "uri", "version", "remote_ip", - "body") + attrs = ("protocol", "host", "method", "uri", "version", "remote_ip") args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) return "%s(%s, headers=%s)" % ( self.__class__.__name__, args, dict(self.headers)) diff -Nru python-tornado-2.1.0/tornado/httputil.py python-tornado-3.1.1/tornado/httputil.py --- python-tornado-2.1.0/tornado/httputil.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/httputil.py 2013-08-04 19:34:21.000000000 +0000 @@ -16,21 +16,75 @@ """HTTP utility code shared by clients and servers.""" -import logging -import urllib -import re +from __future__ import absolute_import, division, print_function, with_statement + +import calendar +import collections +import datetime +import email.utils +import numbers +import time + +from tornado.escape import native_str, parse_qs_bytes, utf8 +from tornado.log import gen_log +from tornado.util import ObjectDict + +try: + from httplib import responses # py2 +except ImportError: + from http.client import responses # py3 + +# responses is unused in this file, but we re-export it to other files. +# Reference it so pyflakes doesn't complain. +responses + +try: + from urllib import urlencode # py2 +except ImportError: + from urllib.parse import urlencode # py3 + + +class _NormalizedHeaderCache(dict): + """Dynamic cached mapping of header names to Http-Header-Case. + + Implemented as a dict subclass so that cache hits are as fast as a + normal dict lookup, without the overhead of a python function + call. + + >>> normalized_headers = _NormalizedHeaderCache(10) + >>> normalized_headers["coNtent-TYPE"] + 'Content-Type' + """ + def __init__(self, size): + super(_NormalizedHeaderCache, self).__init__() + self.size = size + self.queue = collections.deque() + + def __missing__(self, key): + normalized = "-".join([w.capitalize() for w in key.split("-")]) + self[key] = normalized + self.queue.append(key) + if len(self.queue) > self.size: + # Limit the size of the cache. LRU would be better, but this + # simpler approach should be fine. In Python 2.7+ we could + # use OrderedDict (or in 3.2+, @functools.lru_cache). + old_key = self.queue.popleft() + del self[old_key] + return normalized + +_normalized_headers = _NormalizedHeaderCache(1000) -from tornado.util import b, ObjectDict class HTTPHeaders(dict): - """A dictionary that maintains Http-Header-Case for all keys. + """A dictionary that maintains ``Http-Header-Case`` for all keys. Supports multiple values per key via a pair of new methods, - add() and get_list(). The regular dictionary interface returns a single - value per key, with multiple values joined by a comma. + `add()` and `get_list()`. The regular dictionary interface + returns a single value per key, with multiple values joined by a + comma. >>> h = HTTPHeaders({"content-type": "text/html"}) - >>> h.keys() + >>> list(h.keys()) ['Content-Type'] >>> h["Content-Type"] 'text/html' @@ -43,7 +97,7 @@ ['A=B', 'C=D'] >>> for (k,v) in sorted(h.get_all()): - ... print '%s: %s' % (k,v) + ... print('%s: %s' % (k,v)) ... Content-Type: text/html Set-Cookie: A=B @@ -55,24 +109,33 @@ dict.__init__(self) self._as_list = {} self._last_key = None - self.update(*args, **kwargs) + if (len(args) == 1 and len(kwargs) == 0 and + isinstance(args[0], HTTPHeaders)): + # Copy constructor + for k, v in args[0].get_all(): + self.add(k, v) + else: + # Dict-style initialization + self.update(*args, **kwargs) # new public methods def add(self, name, value): """Adds a new value for the given key.""" - norm_name = HTTPHeaders._normalize_name(name) + norm_name = _normalized_headers[name] self._last_key = norm_name if norm_name in self: # bypass our override of __setitem__ since it modifies _as_list - dict.__setitem__(self, norm_name, self[norm_name] + ',' + value) + dict.__setitem__(self, norm_name, + native_str(self[norm_name]) + ',' + + native_str(value)) self._as_list[norm_name].append(value) else: self[norm_name] = value def get_list(self, name): """Returns all values for the given header as a list.""" - norm_name = HTTPHeaders._normalize_name(name) + norm_name = _normalized_headers[name] return self._as_list.get(norm_name, []) def get_all(self): @@ -81,8 +144,8 @@ If a header has multiple values, multiple pairs will be returned with the same name. """ - for name, list in self._as_list.iteritems(): - for value in list: + for name, values in self._as_list.items(): + for value in values: yield (name, value) def parse_line(self, line): @@ -108,7 +171,7 @@ """Returns a dictionary from HTTP header text. >>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n") - >>> sorted(h.iteritems()) + >>> sorted(h.items()) [('Content-Length', '42'), ('Content-Type', 'text/html')] """ h = cls() @@ -120,49 +183,33 @@ # dict implementation overrides def __setitem__(self, name, value): - norm_name = HTTPHeaders._normalize_name(name) + norm_name = _normalized_headers[name] dict.__setitem__(self, norm_name, value) self._as_list[norm_name] = [value] def __getitem__(self, name): - return dict.__getitem__(self, HTTPHeaders._normalize_name(name)) + return dict.__getitem__(self, _normalized_headers[name]) def __delitem__(self, name): - norm_name = HTTPHeaders._normalize_name(name) + norm_name = _normalized_headers[name] dict.__delitem__(self, norm_name) del self._as_list[norm_name] def __contains__(self, name): - norm_name = HTTPHeaders._normalize_name(name) + norm_name = _normalized_headers[name] return dict.__contains__(self, norm_name) def get(self, name, default=None): - return dict.get(self, HTTPHeaders._normalize_name(name), default) + return dict.get(self, _normalized_headers[name], default) def update(self, *args, **kwargs): # dict.update bypasses our __setitem__ - for k, v in dict(*args, **kwargs).iteritems(): + for k, v in dict(*args, **kwargs).items(): self[k] = v - _NORMALIZED_HEADER_RE = re.compile(r'^[A-Z0-9][a-z0-9]*(-[A-Z0-9][a-z0-9]*)*$') - _normalized_headers = {} - - @staticmethod - def _normalize_name(name): - """Converts a name to Http-Header-Case. - - >>> HTTPHeaders._normalize_name("coNtent-TYPE") - 'Content-Type' - """ - try: - return HTTPHeaders._normalized_headers[name] - except KeyError: - if HTTPHeaders._NORMALIZED_HEADER_RE.match(name): - normalized = name - else: - normalized = "-".join([w.capitalize() for w in name.split("-")]) - HTTPHeaders._normalized_headers[name] = normalized - return normalized + def copy(self): + # default implementation returns dict(self), not the subclass + return HTTPHeaders(self) def url_concat(url, args): @@ -172,28 +219,126 @@ >>> url_concat("http://example.com/foo?a=b", dict(c="d")) 'http://example.com/foo?a=b&c=d' """ - if not args: return url + if not args: + return url if url[-1] not in ('?', '&'): url += '&' if ('?' in url) else '?' - return url + urllib.urlencode(args) + return url + urlencode(args) class HTTPFile(ObjectDict): - """Represents an HTTP file. For backwards compatibility, its instance - attributes are also accessible as dictionary keys. + """Represents a file uploaded via a form. - :ivar filename: - :ivar body: - :ivar content_type: The content_type comes from the provided HTTP header - and should not be trusted outright given that it can be easily forged. + For backwards compatibility, its instance attributes are also + accessible as dictionary keys. + + * ``filename`` + * ``body`` + * ``content_type`` """ pass +def _parse_request_range(range_header): + """Parses a Range header. + + Returns either ``None`` or tuple ``(start, end)``. + Note that while the HTTP headers use inclusive byte positions, + this method returns indexes suitable for use in slices. + + >>> start, end = _parse_request_range("bytes=1-2") + >>> start, end + (1, 3) + >>> [0, 1, 2, 3, 4][start:end] + [1, 2] + >>> _parse_request_range("bytes=6-") + (6, None) + >>> _parse_request_range("bytes=-6") + (-6, None) + >>> _parse_request_range("bytes=-0") + (None, 0) + >>> _parse_request_range("bytes=") + (None, None) + >>> _parse_request_range("foo=42") + >>> _parse_request_range("bytes=1-2,6-10") + + Note: only supports one range (ex, ``bytes=1-2,6-10`` is not allowed). + + See [0] for the details of the range header. + + [0]: http://greenbytes.de/tech/webdav/draft-ietf-httpbis-p5-range-latest.html#byte.ranges + """ + unit, _, value = range_header.partition("=") + unit, value = unit.strip(), value.strip() + if unit != "bytes": + return None + start_b, _, end_b = value.partition("-") + try: + start = _int_or_none(start_b) + end = _int_or_none(end_b) + except ValueError: + return None + if end is not None: + if start is None: + if end != 0: + start = -end + end = None + else: + end += 1 + return (start, end) + + +def _get_content_range(start, end, total): + """Returns a suitable Content-Range header: + + >>> print(_get_content_range(None, 1, 4)) + bytes 0-0/4 + >>> print(_get_content_range(1, 3, 4)) + bytes 1-2/4 + >>> print(_get_content_range(None, None, 4)) + bytes 0-3/4 + """ + start = start or 0 + end = (end or total) - 1 + return "bytes %s-%s/%s" % (start, end, total) + + +def _int_or_none(val): + val = val.strip() + if val == "": + return None + return int(val) + + +def parse_body_arguments(content_type, body, arguments, files): + """Parses a form request body. + + Supports ``application/x-www-form-urlencoded`` and + ``multipart/form-data``. The ``content_type`` parameter should be + a string and ``body`` should be a byte string. The ``arguments`` + and ``files`` parameters are dictionaries that will be updated + with the parsed contents. + """ + if content_type.startswith("application/x-www-form-urlencoded"): + uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + for name, values in uri_arguments.items(): + if values: + arguments.setdefault(name, []).extend(values) + elif content_type.startswith("multipart/form-data"): + fields = content_type.split(";") + for field in fields: + k, sep, v = field.strip().partition("=") + if k == "boundary" and v: + parse_multipart_form_data(utf8(v), body, arguments, files) + break + else: + gen_log.warning("Invalid multipart/form-data") + + def parse_multipart_form_data(boundary, data, arguments, files): - """Parses a multipart/form-data body. + """Parses a ``multipart/form-data`` body. - The boundary and data parameters are both byte strings. + The ``boundary`` and ``data`` parameters are both byte strings. The dictionaries given in the arguments and files parameters will be updated with the contents of the body. """ @@ -202,28 +347,29 @@ # xmpp). I think we're also supposed to handle backslash-escapes # here but I'll save that until we see a client that uses them # in the wild. - if boundary.startswith(b('"')) and boundary.endswith(b('"')): + if boundary.startswith(b'"') and boundary.endswith(b'"'): boundary = boundary[1:-1] - if data.endswith(b("\r\n")): - footer_length = len(boundary) + 6 - else: - footer_length = len(boundary) + 4 - parts = data[:-footer_length].split(b("--") + boundary + b("\r\n")) + final_boundary_index = data.rfind(b"--" + boundary + b"--") + if final_boundary_index == -1: + gen_log.warning("Invalid multipart/form-data: no final boundary") + return + parts = data[:final_boundary_index].split(b"--" + boundary + b"\r\n") for part in parts: - if not part: continue - eoh = part.find(b("\r\n\r\n")) + if not part: + continue + eoh = part.find(b"\r\n\r\n") if eoh == -1: - logging.warning("multipart/form-data missing headers") + gen_log.warning("multipart/form-data missing headers") continue headers = HTTPHeaders.parse(part[:eoh].decode("utf-8")) disp_header = headers.get("Content-Disposition", "") disposition, disp_params = _parse_header(disp_header) - if disposition != "form-data" or not part.endswith(b("\r\n")): - logging.warning("Invalid multipart/form-data") + if disposition != "form-data" or not part.endswith(b"\r\n"): + gen_log.warning("Invalid multipart/form-data") continue value = part[eoh + 4:-2] if not disp_params.get("name"): - logging.warning("multipart/form-data value missing name") + gen_log.warning("multipart/form-data value missing name") continue name = disp_params["name"] if disp_params.get("filename"): @@ -235,9 +381,31 @@ arguments.setdefault(name, []).append(value) +def format_timestamp(ts): + """Formats a timestamp in the format used by HTTP. + + The argument may be a numeric timestamp as returned by `time.time`, + a time tuple as returned by `time.gmtime`, or a `datetime.datetime` + object. + + >>> format_timestamp(1359312200) + 'Sun, 27 Jan 2013 18:43:20 GMT' + """ + if isinstance(ts, numbers.Real): + pass + elif isinstance(ts, (tuple, time.struct_time)): + ts = calendar.timegm(ts) + elif isinstance(ts, datetime.datetime): + ts = calendar.timegm(ts.utctimetuple()) + else: + raise TypeError("unknown timestamp type: %r" % ts) + return email.utils.formatdate(ts, usegmt=True) + # _parseparam and _parse_header are copied and modified from python2.7's cgi.py # The original 2.7 version of this code did not correctly support some # combinations of semicolons and double quotes. + + def _parseparam(s): while s[:1] == ';': s = s[1:] @@ -250,6 +418,7 @@ yield f.strip() s = s[end:] + def _parse_header(line): """Parse a Content-type like header. @@ -257,13 +426,13 @@ """ parts = _parseparam(';' + line) - key = parts.next() + key = next(parts) pdict = {} for p in parts: i = p.find('=') if i >= 0: name = p[:i].strip().lower() - value = p[i+1:].strip() + value = p[i + 1:].strip() if len(value) >= 2 and value[0] == value[-1] == '"': value = value[1:-1] value = value.replace('\\\\', '\\').replace('\\"', '"') @@ -274,7 +443,3 @@ def doctests(): import doctest return doctest.DocTestSuite() - -if __name__ == "__main__": - import doctest - doctest.testmod() diff -Nru python-tornado-2.1.0/tornado/ioloop.py python-tornado-3.1.1/tornado/ioloop.py --- python-tornado-2.1.0/tornado/ioloop.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/ioloop.py 2013-09-01 18:41:35.000000000 +0000 @@ -26,37 +26,51 @@ `IOLoop.add_timeout` is a non-blocking alternative to `time.sleep`. """ -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, with_statement import datetime import errno +import functools import heapq -import os import logging +import numbers +import os import select -import thread +import sys import threading import time import traceback +from tornado.concurrent import Future, TracebackFuture +from tornado.log import app_log, gen_log from tornado import stack_context +from tornado.util import Configurable try: import signal except ImportError: signal = None +try: + import thread # py2 +except ImportError: + import _thread as thread # py3 + from tornado.platform.auto import set_close_exec, Waker -class IOLoop(object): +class TimeoutError(Exception): + pass + + +class IOLoop(Configurable): """A level-triggered I/O loop. - We use epoll (Linux) or kqueue (BSD and Mac OS X; requires python - 2.6+) if they are available, or else we fall back on select(). If - you are implementing a system that needs to handle thousands of - simultaneous connections, you should use a system that supports either - epoll or queue. + We use ``epoll`` (Linux) or ``kqueue`` (BSD and Mac OS X) if they + are available, or else we fall back on select(). If you are + implementing a system that needs to handle thousands of + simultaneous connections, you should use a system that supports + either ``epoll`` or ``kqueue``. Example usage for a simple TCP server:: @@ -102,47 +116,26 @@ NONE = 0 READ = _EPOLLIN WRITE = _EPOLLOUT - ERROR = _EPOLLERR | _EPOLLHUP | _EPOLLRDHUP + ERROR = _EPOLLERR | _EPOLLHUP - def __init__(self, impl=None): - self._impl = impl or _poll() - if hasattr(self._impl, 'fileno'): - set_close_exec(self._impl.fileno()) - self._handlers = {} - self._events = {} - self._callbacks = [] - self._callback_lock = threading.Lock() - self._timeouts = [] - self._running = False - self._stopped = False - self._thread_ident = None - self._blocking_signal_threshold = None + # Global lock for creating global IOLoop instance + _instance_lock = threading.Lock() - # Create a pipe that we send bogus data to when we want to wake - # the I/O loop when it is idle - self._waker = Waker() - self.add_handler(self._waker.fileno(), - lambda fd, events: self._waker.consume(), - self.READ) + _current = threading.local() @staticmethod def instance(): - """Returns a global IOLoop instance. + """Returns a global `IOLoop` instance. - Most single-threaded applications have a single, global IOLoop. - Use this method instead of passing around IOLoop instances - throughout your code. - - A common pattern for classes that depend on IOLoops is to use - a default argument to enable programs with multiple IOLoops - but not require the argument for simpler applications:: - - class MyClass(object): - def __init__(self, io_loop=None): - self.io_loop = io_loop or IOLoop.instance() + Most applications have a single, global `IOLoop` running on the + main thread. Use this method to get this instance from + another thread. To get the current thread's `IOLoop`, use `current()`. """ if not hasattr(IOLoop, "_instance"): - IOLoop._instance = IOLoop() + with IOLoop._instance_lock: + if not hasattr(IOLoop, "_instance"): + # New instance after double check + IOLoop._instance = IOLoop() return IOLoop._instance @staticmethod @@ -151,98 +144,459 @@ return hasattr(IOLoop, "_instance") def install(self): - """Installs this IOloop object as the singleton instance. + """Installs this `IOLoop` object as the singleton instance. This is normally not necessary as `instance()` will create - an IOLoop on demand, but you may want to call `install` to use - a custom subclass of IOLoop. + an `IOLoop` on demand, but you may want to call `install` to use + a custom subclass of `IOLoop`. """ assert not IOLoop.initialized() IOLoop._instance = self + @staticmethod + def current(): + """Returns the current thread's `IOLoop`. + + If an `IOLoop` is currently running or has been marked as current + by `make_current`, returns that instance. Otherwise returns + `IOLoop.instance()`, i.e. the main thread's `IOLoop`. + + A common pattern for classes that depend on ``IOLoops`` is to use + a default argument to enable programs with multiple ``IOLoops`` + but not require the argument for simpler applications:: + + class MyClass(object): + def __init__(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + + In general you should use `IOLoop.current` as the default when + constructing an asynchronous object, and use `IOLoop.instance` + when you mean to communicate to the main thread from a different + one. + """ + current = getattr(IOLoop._current, "instance", None) + if current is None: + return IOLoop.instance() + return current + + def make_current(self): + """Makes this the `IOLoop` for the current thread. + + An `IOLoop` automatically becomes current for its thread + when it is started, but it is sometimes useful to call + `make_current` explictly before starting the `IOLoop`, + so that code run at startup time can find the right + instance. + """ + IOLoop._current.instance = self + + @staticmethod + def clear_current(): + IOLoop._current.instance = None + + @classmethod + def configurable_base(cls): + return IOLoop + + @classmethod + def configurable_default(cls): + if hasattr(select, "epoll"): + from tornado.platform.epoll import EPollIOLoop + return EPollIOLoop + if hasattr(select, "kqueue"): + # Python 2.6+ on BSD or Mac + from tornado.platform.kqueue import KQueueIOLoop + return KQueueIOLoop + from tornado.platform.select import SelectIOLoop + return SelectIOLoop + + def initialize(self): + pass + def close(self, all_fds=False): - """Closes the IOLoop, freeing any resources used. + """Closes the `IOLoop`, freeing any resources used. If ``all_fds`` is true, all file descriptors registered on the - IOLoop will be closed (not just the ones created by the IOLoop itself. + IOLoop will be closed (not just the ones created by the + `IOLoop` itself). + + Many applications will only use a single `IOLoop` that runs for the + entire lifetime of the process. In that case closing the `IOLoop` + is not necessary since everything will be cleaned up when the + process exits. `IOLoop.close` is provided mainly for scenarios + such as unit tests, which create and destroy a large number of + ``IOLoops``. + + An `IOLoop` must be completely stopped before it can be closed. This + means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must + be allowed to return before attempting to call `IOLoop.close()`. + Therefore the call to `close` will usually appear just after + the call to `start` rather than near the call to `stop`. + + .. versionchanged:: 3.1 + If the `IOLoop` implementation supports non-integer objects + for "file descriptors", those objects will have their + ``close`` method when ``all_fds`` is true. """ - self.remove_handler(self._waker.fileno()) - if all_fds: - for fd in self._handlers.keys()[:]: - try: - os.close(fd) - except Exception: - logging.debug("error closing fd %d", fd, exc_info=True) - self._waker.close() - self._impl.close() + raise NotImplementedError() def add_handler(self, fd, handler, events): - """Registers the given handler to receive the given events for fd.""" - self._handlers[fd] = stack_context.wrap(handler) - self._impl.register(fd, events | self.ERROR) + """Registers the given handler to receive the given events for fd. + + The ``events`` argument is a bitwise or of the constants + ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``. + + When an event occurs, ``handler(fd, events)`` will be run. + """ + raise NotImplementedError() def update_handler(self, fd, events): """Changes the events we listen for fd.""" - self._impl.modify(fd, events | self.ERROR) + raise NotImplementedError() def remove_handler(self, fd): """Stop listening for events on fd.""" - self._handlers.pop(fd, None) - self._events.pop(fd, None) - try: - self._impl.unregister(fd) - except (OSError, IOError): - logging.debug("Error deleting fd from IOLoop", exc_info=True) + raise NotImplementedError() def set_blocking_signal_threshold(self, seconds, action): - """Sends a signal if the ioloop is blocked for more than s seconds. + """Sends a signal if the `IOLoop` is blocked for more than + ``s`` seconds. - Pass seconds=None to disable. Requires python 2.6 on a unixy + Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy platform. - The action parameter is a python signal handler. Read the - documentation for the python 'signal' module for more information. - If action is None, the process will be killed if it is blocked for - too long. + The action parameter is a Python signal handler. Read the + documentation for the `signal` module for more information. + If ``action`` is None, the process will be killed if it is + blocked for too long. """ - if not hasattr(signal, "setitimer"): - logging.error("set_blocking_signal_threshold requires a signal module " - "with the setitimer method") - return - self._blocking_signal_threshold = seconds - if seconds is not None: - signal.signal(signal.SIGALRM, - action if action is not None else signal.SIG_DFL) + raise NotImplementedError() def set_blocking_log_threshold(self, seconds): - """Logs a stack trace if the ioloop is blocked for more than s seconds. - Equivalent to set_blocking_signal_threshold(seconds, self.log_stack) + """Logs a stack trace if the `IOLoop` is blocked for more than + ``s`` seconds. + + Equivalent to ``set_blocking_signal_threshold(seconds, + self.log_stack)`` """ self.set_blocking_signal_threshold(seconds, self.log_stack) def log_stack(self, signal, frame): """Signal handler to log the stack trace of the current thread. - For use with set_blocking_signal_threshold. + For use with `set_blocking_signal_threshold`. """ - logging.warning('IOLoop blocked for %f seconds in\n%s', + gen_log.warning('IOLoop blocked for %f seconds in\n%s', self._blocking_signal_threshold, ''.join(traceback.format_stack(frame))) def start(self): """Starts the I/O loop. - The loop will run until one of the I/O handlers calls stop(), which + The loop will run until one of the callbacks calls `stop()`, which will make the loop stop after the current event iteration completes. """ + raise NotImplementedError() + + def stop(self): + """Stop the I/O loop. + + If the event loop is not currently running, the next call to `start()` + will return immediately. + + To use asynchronous methods from otherwise-synchronous code (such as + unit tests), you can start and stop the event loop like this:: + + ioloop = IOLoop() + async_method(ioloop=ioloop, callback=ioloop.stop) + ioloop.start() + + ``ioloop.start()`` will return after ``async_method`` has run + its callback, whether that callback was invoked before or + after ``ioloop.start``. + + Note that even after `stop` has been called, the `IOLoop` is not + completely stopped until `IOLoop.start` has also returned. + Some work that was scheduled before the call to `stop` may still + be run before the `IOLoop` shuts down. + """ + raise NotImplementedError() + + def run_sync(self, func, timeout=None): + """Starts the `IOLoop`, runs the given function, and stops the loop. + + If the function returns a `.Future`, the `IOLoop` will run + until the future is resolved. If it raises an exception, the + `IOLoop` will stop and the exception will be re-raised to the + caller. + + The keyword-only argument ``timeout`` may be used to set + a maximum duration for the function. If the timeout expires, + a `TimeoutError` is raised. + + This method is useful in conjunction with `tornado.gen.coroutine` + to allow asynchronous calls in a ``main()`` function:: + + @gen.coroutine + def main(): + # do stuff... + + if __name__ == '__main__': + IOLoop.instance().run_sync(main) + """ + future_cell = [None] + + def run(): + try: + result = func() + except Exception: + future_cell[0] = TracebackFuture() + future_cell[0].set_exc_info(sys.exc_info()) + else: + if isinstance(result, Future): + future_cell[0] = result + else: + future_cell[0] = Future() + future_cell[0].set_result(result) + self.add_future(future_cell[0], lambda future: self.stop()) + self.add_callback(run) + if timeout is not None: + timeout_handle = self.add_timeout(self.time() + timeout, self.stop) + self.start() + if timeout is not None: + self.remove_timeout(timeout_handle) + if not future_cell[0].done(): + raise TimeoutError('Operation timed out after %s seconds' % timeout) + return future_cell[0].result() + + def time(self): + """Returns the current time according to the `IOLoop`'s clock. + + The return value is a floating-point number relative to an + unspecified time in the past. + + By default, the `IOLoop`'s time function is `time.time`. However, + it may be configured to use e.g. `time.monotonic` instead. + Calls to `add_timeout` that pass a number instead of a + `datetime.timedelta` should use this function to compute the + appropriate time, so they can work no matter what time function + is chosen. + """ + return time.time() + + def add_timeout(self, deadline, callback): + """Runs the ``callback`` at the time ``deadline`` from the I/O loop. + + Returns an opaque handle that may be passed to + `remove_timeout` to cancel. + + ``deadline`` may be a number denoting a time (on the same + scale as `IOLoop.time`, normally `time.time`), or a + `datetime.timedelta` object for a deadline relative to the + current time. + + Note that it is not safe to call `add_timeout` from other threads. + Instead, you must use `add_callback` to transfer control to the + `IOLoop`'s thread, and then call `add_timeout` from there. + """ + raise NotImplementedError() + + def remove_timeout(self, timeout): + """Cancels a pending timeout. + + The argument is a handle as returned by `add_timeout`. It is + safe to call `remove_timeout` even if the callback has already + been run. + """ + raise NotImplementedError() + + def add_callback(self, callback, *args, **kwargs): + """Calls the given callback on the next I/O loop iteration. + + It is safe to call this method from any thread at any time, + except from a signal handler. Note that this is the **only** + method in `IOLoop` that makes this thread-safety guarantee; all + other interaction with the `IOLoop` must be done from that + `IOLoop`'s thread. `add_callback()` may be used to transfer + control from other threads to the `IOLoop`'s thread. + + To add a callback from a signal handler, see + `add_callback_from_signal`. + """ + raise NotImplementedError() + + def add_callback_from_signal(self, callback, *args, **kwargs): + """Calls the given callback on the next I/O loop iteration. + + Safe for use from a Python signal handler; should not be used + otherwise. + + Callbacks added with this method will be run without any + `.stack_context`, to avoid picking up the context of the function + that was interrupted by the signal. + """ + raise NotImplementedError() + + def add_future(self, future, callback): + """Schedules a callback on the ``IOLoop`` when the given + `.Future` is finished. + + The callback is invoked with one argument, the + `.Future`. + """ + assert isinstance(future, Future) + callback = stack_context.wrap(callback) + future.add_done_callback( + lambda future: self.add_callback(callback, future)) + + def _run_callback(self, callback): + """Runs a callback with error handling. + + For use in subclasses. + """ + try: + callback() + except Exception: + self.handle_callback_exception(callback) + + def handle_callback_exception(self, callback): + """This method is called whenever a callback run by the `IOLoop` + throws an exception. + + By default simply logs the exception as an error. Subclasses + may override this method to customize reporting of exceptions. + + The exception itself is not passed explicitly, but is available + in `sys.exc_info`. + """ + app_log.error("Exception in callback %r", callback, exc_info=True) + + +class PollIOLoop(IOLoop): + """Base class for IOLoops built around a select-like function. + + For concrete implementations, see `tornado.platform.epoll.EPollIOLoop` + (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or + `tornado.platform.select.SelectIOLoop` (all platforms). + """ + def initialize(self, impl, time_func=None): + super(PollIOLoop, self).initialize() + self._impl = impl + if hasattr(self._impl, 'fileno'): + set_close_exec(self._impl.fileno()) + self.time_func = time_func or time.time + self._handlers = {} + self._events = {} + self._callbacks = [] + self._callback_lock = threading.Lock() + self._timeouts = [] + self._cancellations = 0 + self._running = False + self._stopped = False + self._closing = False + self._thread_ident = None + self._blocking_signal_threshold = None + + # Create a pipe that we send bogus data to when we want to wake + # the I/O loop when it is idle + self._waker = Waker() + self.add_handler(self._waker.fileno(), + lambda fd, events: self._waker.consume(), + self.READ) + + def close(self, all_fds=False): + with self._callback_lock: + self._closing = True + self.remove_handler(self._waker.fileno()) + if all_fds: + for fd in self._handlers.keys(): + try: + close_method = getattr(fd, 'close', None) + if close_method is not None: + close_method() + else: + os.close(fd) + except Exception: + gen_log.debug("error closing fd %s", fd, exc_info=True) + self._waker.close() + self._impl.close() + + def add_handler(self, fd, handler, events): + self._handlers[fd] = stack_context.wrap(handler) + self._impl.register(fd, events | self.ERROR) + + def update_handler(self, fd, events): + self._impl.modify(fd, events | self.ERROR) + + def remove_handler(self, fd): + self._handlers.pop(fd, None) + self._events.pop(fd, None) + try: + self._impl.unregister(fd) + except Exception: + gen_log.debug("Error deleting fd from IOLoop", exc_info=True) + + def set_blocking_signal_threshold(self, seconds, action): + if not hasattr(signal, "setitimer"): + gen_log.error("set_blocking_signal_threshold requires a signal module " + "with the setitimer method") + return + self._blocking_signal_threshold = seconds + if seconds is not None: + signal.signal(signal.SIGALRM, + action if action is not None else signal.SIG_DFL) + + def start(self): + if not logging.getLogger().handlers: + # The IOLoop catches and logs exceptions, so it's + # important that log output be visible. However, python's + # default behavior for non-root loggers (prior to python + # 3.2) is to print an unhelpful "no handlers could be + # found" message rather than the actual log entry, so we + # must explicitly configure logging if we've made it this + # far without anything. + logging.basicConfig() if self._stopped: self._stopped = False return + old_current = getattr(IOLoop._current, "instance", None) + IOLoop._current.instance = self self._thread_ident = thread.get_ident() self._running = True + + # signal.set_wakeup_fd closes a race condition in event loops: + # a signal may arrive at the beginning of select/poll/etc + # before it goes into its interruptible sleep, so the signal + # will be consumed without waking the select. The solution is + # for the (C, synchronous) signal handler to write to a pipe, + # which will then be seen by select. + # + # In python's signal handling semantics, this only matters on the + # main thread (fortunately, set_wakeup_fd only works on the main + # thread and will raise a ValueError otherwise). + # + # If someone has already set a wakeup fd, we don't want to + # disturb it. This is an issue for twisted, which does its + # SIGCHILD processing in response to its own wakeup fd being + # written to. As long as the wakeup fd is registered on the IOLoop, + # the loop will still wake up and everything should work. + old_wakeup_fd = None + if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': + # requires python 2.6+, unix. set_wakeup_fd exists but crashes + # the python process on windows. + try: + old_wakeup_fd = signal.set_wakeup_fd(self._waker.write_fileno()) + if old_wakeup_fd != -1: + # Already set, restore previous value. This is a little racy, + # but there's no clean get_wakeup_fd and in real use the + # IOLoop is just started once at the beginning. + signal.set_wakeup_fd(old_wakeup_fd) + old_wakeup_fd = None + except ValueError: # non-main thread + pass + while True: - # Never use an infinite timeout here - it can stall epoll - poll_timeout = 0.2 + poll_timeout = 3600.0 # Prevent IO event starvation by delaying new callbacks # to the next iteration of the event loop. @@ -253,18 +607,27 @@ self._run_callback(callback) if self._timeouts: - now = time.time() + now = self.time() while self._timeouts: if self._timeouts[0].callback is None: # the timeout was cancelled heapq.heappop(self._timeouts) + self._cancellations -= 1 elif self._timeouts[0].deadline <= now: timeout = heapq.heappop(self._timeouts) self._run_callback(timeout.callback) else: - milliseconds = self._timeouts[0].deadline - now - poll_timeout = min(milliseconds, poll_timeout) + seconds = self._timeouts[0].deadline - now + poll_timeout = min(seconds, poll_timeout) break + if (self._cancellations > 512 + and self._cancellations > (len(self._timeouts) >> 1)): + # Clean up the timeout queue when it gets large and it's + # more than half cancellations. + self._cancellations = 0 + self._timeouts = [x for x in self._timeouts + if x.callback is not None] + heapq.heapify(self._timeouts) if self._callbacks: # If any callbacks or timeouts called add_callback, @@ -281,7 +644,7 @@ try: event_pairs = self._impl.poll(poll_timeout) - except Exception, e: + except Exception as e: # Depending on python version and IOLoop implementation, # different exception types may be thrown and there are # two ways EINTR might be signaled: @@ -307,81 +670,53 @@ fd, events = self._events.popitem() try: self._handlers[fd](fd, events) - except (OSError, IOError), e: + except (OSError, IOError) as e: if e.args[0] == errno.EPIPE: # Happens when the client closes the connection pass else: - logging.error("Exception in I/O handler for fd %d", + app_log.error("Exception in I/O handler for fd %s", fd, exc_info=True) except Exception: - logging.error("Exception in I/O handler for fd %d", + app_log.error("Exception in I/O handler for fd %s", fd, exc_info=True) # reset the stopped flag so another start/stop pair can be issued self._stopped = False if self._blocking_signal_threshold is not None: signal.setitimer(signal.ITIMER_REAL, 0, 0) + IOLoop._current.instance = old_current + if old_wakeup_fd is not None: + signal.set_wakeup_fd(old_wakeup_fd) def stop(self): - """Stop the loop after the current event loop iteration is complete. - If the event loop is not currently running, the next call to start() - will return immediately. - - To use asynchronous methods from otherwise-synchronous code (such as - unit tests), you can start and stop the event loop like this:: - - ioloop = IOLoop() - async_method(ioloop=ioloop, callback=ioloop.stop) - ioloop.start() - - ioloop.start() will return after async_method has run its callback, - whether that callback was invoked before or after ioloop.start. - """ self._running = False self._stopped = True self._waker.wake() - def running(self): - """Returns true if this IOLoop is currently running.""" - return self._running + def time(self): + return self.time_func() def add_timeout(self, deadline, callback): - """Calls the given callback at the time deadline from the I/O loop. - - Returns a handle that may be passed to remove_timeout to cancel. - - ``deadline`` may be a number denoting a unix timestamp (as returned - by ``time.time()`` or a ``datetime.timedelta`` object for a deadline - relative to the current time. - """ - timeout = _Timeout(deadline, stack_context.wrap(callback)) + timeout = _Timeout(deadline, stack_context.wrap(callback), self) heapq.heappush(self._timeouts, timeout) return timeout def remove_timeout(self, timeout): - """Cancels a pending timeout. - - The argument is a handle as returned by add_timeout. - """ # Removing from a heap is complicated, so just leave the defunct # timeout object in the queue (see discussion in # http://docs.python.org/library/heapq.html). # If this turns out to be a problem, we could add a garbage # collection pass whenever there are too many dead timeouts. timeout.callback = None + self._cancellations += 1 - def add_callback(self, callback): - """Calls the given callback on the next I/O loop iteration. - - It is safe to call this method from any thread at any time. - Note that this is the *only* method in IOLoop that makes this - guarantee; all other interaction with the IOLoop must be done - from that IOLoop's thread. add_callback() may be used to transfer - control from other threads to the IOLoop's thread. - """ + def add_callback(self, callback, *args, **kwargs): with self._callback_lock: + if self._closing: + raise RuntimeError("IOLoop is closing") list_empty = not self._callbacks - self._callbacks.append(stack_context.wrap(callback)) + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) if list_empty and thread.get_ident() != self._thread_ident: # If we're in the IOLoop's thread, we know it's not currently # polling. If we're not, and we added the first callback to an @@ -391,23 +726,23 @@ # avoid it when we can. self._waker.wake() - def _run_callback(self, callback): - try: - callback() - except Exception: - self.handle_callback_exception(callback) - - def handle_callback_exception(self, callback): - """This method is called whenever a callback run by the IOLoop - throws an exception. - - By default simply logs the exception as an error. Subclasses - may override this method to customize reporting of exceptions. - - The exception itself is not passed explicitly, but is available - in sys.exc_info. - """ - logging.error("Exception in callback %r", callback, exc_info=True) + def add_callback_from_signal(self, callback, *args, **kwargs): + with stack_context.NullContext(): + if thread.get_ident() != self._thread_ident: + # if the signal is handled on another thread, we can add + # it normally (modulo the NullContext) + self.add_callback(callback, *args, **kwargs) + else: + # If we're on the IOLoop's thread, we cannot use + # the regular add_callback because it may deadlock on + # _callback_lock. Blindly insert into self._callbacks. + # This is safe because the GIL makes list.append atomic. + # One subtlety is that if the signal interrupted the + # _callback_lock block in IOLoop.start, we may modify + # either the old or new version of self._callbacks, + # but either way will work. + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) class _Timeout(object): @@ -416,11 +751,11 @@ # Reduce memory overhead when there are lots of pending callbacks __slots__ = ['deadline', 'callback'] - def __init__(self, deadline, callback): - if isinstance(deadline, (int, long, float)): + def __init__(self, deadline, callback, io_loop): + if isinstance(deadline, numbers.Real): self.deadline = deadline elif isinstance(deadline, datetime.timedelta): - self.deadline = time.time() + _Timeout.timedelta_to_seconds(deadline) + self.deadline = io_loop.time() + _Timeout.timedelta_to_seconds(deadline) else: raise TypeError("Unsupported deadline %r" % deadline) self.callback = callback @@ -428,7 +763,7 @@ @staticmethod def timedelta_to_seconds(td): """Equivalent to td.total_seconds() (introduced in python 2.7).""" - return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / float(10**6) + return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) # Comparison methods to sort by deadline, with object id as a tiebreaker # to guarantee a consistent ordering. The heapq module uses __le__ @@ -446,191 +781,44 @@ class PeriodicCallback(object): """Schedules the given callback to be called periodically. - The callback is called every callback_time milliseconds. + The callback is called every ``callback_time`` milliseconds. - `start` must be called after the PeriodicCallback is created. + `start` must be called after the `PeriodicCallback` is created. """ def __init__(self, callback, callback_time, io_loop=None): self.callback = callback + if callback_time <= 0: + raise ValueError("Periodic callback must have a positive callback_time") self.callback_time = callback_time - self.io_loop = io_loop or IOLoop.instance() + self.io_loop = io_loop or IOLoop.current() self._running = False + self._timeout = None def start(self): """Starts the timer.""" self._running = True - self._next_timeout = time.time() + self._next_timeout = self.io_loop.time() self._schedule_next() def stop(self): """Stops the timer.""" self._running = False + if self._timeout is not None: + self.io_loop.remove_timeout(self._timeout) + self._timeout = None def _run(self): - if not self._running: return + if not self._running: + return try: self.callback() except Exception: - logging.error("Error in periodic callback", exc_info=True) + app_log.error("Error in periodic callback", exc_info=True) self._schedule_next() def _schedule_next(self): if self._running: - current_time = time.time() - while self._next_timeout < current_time: + current_time = self.io_loop.time() + while self._next_timeout <= current_time: self._next_timeout += self.callback_time / 1000.0 - self.io_loop.add_timeout(self._next_timeout, self._run) - - -class _EPoll(object): - """An epoll-based event loop using our C module for Python 2.5 systems""" - _EPOLL_CTL_ADD = 1 - _EPOLL_CTL_DEL = 2 - _EPOLL_CTL_MOD = 3 - - def __init__(self): - self._epoll_fd = epoll.epoll_create() - - def fileno(self): - return self._epoll_fd - - def close(self): - os.close(self._epoll_fd) - - def register(self, fd, events): - epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events) - - def modify(self, fd, events): - epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events) - - def unregister(self, fd): - epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0) - - def poll(self, timeout): - return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000)) - - -class _KQueue(object): - """A kqueue-based event loop for BSD/Mac systems.""" - def __init__(self): - self._kqueue = select.kqueue() - self._active = {} - - def fileno(self): - return self._kqueue.fileno() - - def close(self): - self._kqueue.close() - - def register(self, fd, events): - self._control(fd, events, select.KQ_EV_ADD) - self._active[fd] = events - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - events = self._active.pop(fd) - self._control(fd, events, select.KQ_EV_DELETE) - - def _control(self, fd, events, flags): - kevents = [] - if events & IOLoop.WRITE: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_WRITE, flags=flags)) - if events & IOLoop.READ or not kevents: - # Always read when there is not a write - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_READ, flags=flags)) - # Even though control() takes a list, it seems to return EINVAL - # on Mac OS X (10.6) when there is more than one event in the list. - for kevent in kevents: - self._kqueue.control([kevent], 0) - - def poll(self, timeout): - kevents = self._kqueue.control(None, 1000, timeout) - events = {} - for kevent in kevents: - fd = kevent.ident - if kevent.filter == select.KQ_FILTER_READ: - events[fd] = events.get(fd, 0) | IOLoop.READ - if kevent.filter == select.KQ_FILTER_WRITE: - if kevent.flags & select.KQ_EV_EOF: - # If an asynchronous connection is refused, kqueue - # returns a write event with the EOF flag set. - # Turn this into an error for consistency with the - # other IOLoop implementations. - # Note that for read events, EOF may be returned before - # all data has been consumed from the socket buffer, - # so we only check for EOF on write events. - events[fd] = IOLoop.ERROR - else: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - if kevent.flags & select.KQ_EV_ERROR: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -class _Select(object): - """A simple, select()-based IOLoop implementation for non-Linux systems""" - def __init__(self): - self.read_fds = set() - self.write_fds = set() - self.error_fds = set() - self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) - - def close(self): - pass - - def register(self, fd, events): - if events & IOLoop.READ: self.read_fds.add(fd) - if events & IOLoop.WRITE: self.write_fds.add(fd) - if events & IOLoop.ERROR: - self.error_fds.add(fd) - # Closed connections are reported as errors by epoll and kqueue, - # but as zero-byte reads by select, so when errors are requested - # we need to listen for both read and error. - self.read_fds.add(fd) - - def modify(self, fd, events): - self.unregister(fd) - self.register(fd, events) - - def unregister(self, fd): - self.read_fds.discard(fd) - self.write_fds.discard(fd) - self.error_fds.discard(fd) - - def poll(self, timeout): - readable, writeable, errors = select.select( - self.read_fds, self.write_fds, self.error_fds, timeout) - events = {} - for fd in readable: - events[fd] = events.get(fd, 0) | IOLoop.READ - for fd in writeable: - events[fd] = events.get(fd, 0) | IOLoop.WRITE - for fd in errors: - events[fd] = events.get(fd, 0) | IOLoop.ERROR - return events.items() - - -# Choose a poll implementation. Use epoll if it is available, fall back to -# select() for non-Linux platforms -if hasattr(select, "epoll"): - # Python 2.6+ on Linux - _poll = select.epoll -elif hasattr(select, "kqueue"): - # Python 2.6+ on BSD or Mac - _poll = _KQueue -else: - try: - # Linux systems with our C module installed - import epoll - _poll = _EPoll - except Exception: - # All other systems - import sys - if "linux" in sys.platform: - logging.warning("epoll module not found; using select()") - _poll = _Select + self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) diff -Nru python-tornado-2.1.0/tornado/iostream.py python-tornado-3.1.1/tornado/iostream.py --- python-tornado-2.1.0/tornado/iostream.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/iostream.py 2013-09-01 18:41:35.000000000 +0000 @@ -14,75 +14,68 @@ # License for the specific language governing permissions and limitations # under the License. -"""A utility class to write to and read from a non-blocking socket.""" +"""Utility classes to write to and read from non-blocking files and sockets. -from __future__ import with_statement +Contents: + +* `BaseIOStream`: Generic interface for reading and writing. +* `IOStream`: Implementation of BaseIOStream using non-blocking sockets. +* `SSLIOStream`: SSL-aware version of IOStream. +* `PipeIOStream`: Pipe-based IOStream implementation. +""" + +from __future__ import absolute_import, division, print_function, with_statement import collections import errno -import logging +import numbers +import os import socket +import ssl import sys import re from tornado import ioloop +from tornado.log import gen_log, app_log +from tornado.netutil import ssl_wrap_socket, ssl_match_hostname, SSLCertificateError from tornado import stack_context -from tornado.util import b, bytes_type +from tornado.util import bytes_type try: - import ssl # Python 2.6+ + from tornado.platform.posix import _set_nonblocking except ImportError: - ssl = None - -class IOStream(object): - r"""A utility class to write to and read from a non-blocking socket. - - We support a non-blocking ``write()`` and a family of ``read_*()`` methods. - All of the methods take callbacks (since writing and reading are - non-blocking and asynchronous). + _set_nonblocking = None - The socket parameter may either be connected or unconnected. For - server operations the socket is the result of calling socket.accept(). - For client operations the socket is created with socket.socket(), - and may either be connected before passing it to the IOStream or - connected with IOStream.connect. - A very simple (and broken) HTTP client using this class:: +class StreamClosedError(IOError): + """Exception raised by `IOStream` methods when the stream is closed. - from tornado import ioloop - from tornado import iostream - import socket + Note that the close callback is scheduled to run *after* other + callbacks on the stream (to allow for buffered data to be processed), + so you may see this error before you see the close callback. + """ + pass - def send_request(): - stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") - stream.read_until("\r\n\r\n", on_headers) - def on_headers(data): - headers = {} - for line in data.split("\r\n"): - parts = line.split(":") - if len(parts) == 2: - headers[parts[0].strip()] = parts[1].strip() - stream.read_bytes(int(headers["Content-Length"]), on_body) +class BaseIOStream(object): + """A utility class to write to and read from a non-blocking file or socket. - def on_body(data): - print data - stream.close() - ioloop.IOLoop.instance().stop() + We support a non-blocking ``write()`` and a family of ``read_*()`` methods. + All of the methods take callbacks (since writing and reading are + non-blocking and asynchronous). - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - stream = iostream.IOStream(s) - stream.connect(("friendfeed.com", 80), send_request) - ioloop.IOLoop.instance().start() + When a stream is closed due to an error, the IOStream's ``error`` + attribute contains the exception object. + Subclasses must implement `fileno`, `close_fd`, `write_to_fd`, + `read_from_fd`, and optionally `get_fd_error`. """ - def __init__(self, socket, io_loop=None, max_buffer_size=104857600, + def __init__(self, io_loop=None, max_buffer_size=None, read_chunk_size=4096): - self.socket = socket - self.socket.setblocking(False) - self.io_loop = io_loop or ioloop.IOLoop.instance() - self.max_buffer_size = max_buffer_size + self.io_loop = io_loop or ioloop.IOLoop.current() + self.max_buffer_size = max_buffer_size or 104857600 self.read_chunk_size = read_chunk_size + self.error = None self._read_buffer = collections.deque() self._write_buffer = collections.deque() self._read_buffer_size = 0 @@ -99,140 +92,180 @@ self._connecting = False self._state = None self._pending_callbacks = 0 + self._closed = False - def connect(self, address, callback=None): - """Connects the socket to a remote address without blocking. + def fileno(self): + """Returns the file descriptor for this stream.""" + raise NotImplementedError() - May only be called if the socket passed to the constructor was - not previously connected. The address parameter is in the - same format as for socket.connect, i.e. a (host, port) tuple. - If callback is specified, it will be called when the - connection is completed. - - Note that it is safe to call IOStream.write while the - connection is pending, in which case the data will be written - as soon as the connection is ready. Calling IOStream read - methods before the socket is connected works on some platforms - but is non-portable. + def close_fd(self): + """Closes the file underlying this stream. + + ``close_fd`` is called by `BaseIOStream` and should not be called + elsewhere; other users should call `close` instead. """ - self._connecting = True - try: - self.socket.connect(address) - except socket.error, e: - # In non-blocking mode connect() always raises an exception - if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): - raise - self._connect_callback = stack_context.wrap(callback) - self._add_io_state(self.io_loop.WRITE) + raise NotImplementedError() + + def write_to_fd(self, data): + """Attempts to write ``data`` to the underlying file. + + Returns the number of bytes written. + """ + raise NotImplementedError() + + def read_from_fd(self): + """Attempts to read from the underlying file. + + Returns ``None`` if there was nothing to read (the socket + returned `~errno.EWOULDBLOCK` or equivalent), otherwise + returns the data. When possible, should return no more than + ``self.read_chunk_size`` bytes at a time. + """ + raise NotImplementedError() + + def get_fd_error(self): + """Returns information about any error on the underlying file. + + This method is called after the `.IOLoop` has signaled an error on the + file descriptor, and should return an Exception (such as `socket.error` + with additional information, or None if no such information is + available. + """ + return None def read_until_regex(self, regex, callback): - """Call callback when we read the given regex pattern.""" - assert not self._read_callback, "Already reading" + """Run ``callback`` when we read the given regex pattern. + + The callback will get the data read (including the data that + matched the regex and anything that came before it) as an argument. + """ + self._set_read_callback(callback) self._read_regex = re.compile(regex) - self._read_callback = stack_context.wrap(callback) - while True: - # See if we've already got the data from a previous read - if self._read_from_buffer(): - return - self._check_closed() - if self._read_to_buffer() == 0: - break - self._add_io_state(self.io_loop.READ) - + self._try_inline_read() + def read_until(self, delimiter, callback): - """Call callback when we read the given delimiter.""" - assert not self._read_callback, "Already reading" + """Run ``callback`` when we read the given delimiter. + + The callback will get the data read (including the delimiter) + as an argument. + """ + self._set_read_callback(callback) self._read_delimiter = delimiter - self._read_callback = stack_context.wrap(callback) - while True: - # See if we've already got the data from a previous read - if self._read_from_buffer(): - return - self._check_closed() - if self._read_to_buffer() == 0: - break - self._add_io_state(self.io_loop.READ) + self._try_inline_read() def read_bytes(self, num_bytes, callback, streaming_callback=None): - """Call callback when we read the given number of bytes. + """Run callback when we read the given number of bytes. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final - ``callback`` will be empty. + ``callback`` will be empty. Otherwise, the ``callback`` gets + the data as an argument. """ - assert not self._read_callback, "Already reading" - assert isinstance(num_bytes, int) + self._set_read_callback(callback) + assert isinstance(num_bytes, numbers.Integral) self._read_bytes = num_bytes - self._read_callback = stack_context.wrap(callback) self._streaming_callback = stack_context.wrap(streaming_callback) - while True: - if self._read_from_buffer(): - return - self._check_closed() - if self._read_to_buffer() == 0: - break - self._add_io_state(self.io_loop.READ) + self._try_inline_read() def read_until_close(self, callback, streaming_callback=None): """Reads all data from the socket until it is closed. If a ``streaming_callback`` is given, it will be called with chunks of data as they become available, and the argument to the final - ``callback`` will be empty. + ``callback`` will be empty. Otherwise, the ``callback`` gets the + data as an argument. Subject to ``max_buffer_size`` limit from `IOStream` constructor if a ``streaming_callback`` is not used. """ - assert not self._read_callback, "Already reading" + self._set_read_callback(callback) + self._streaming_callback = stack_context.wrap(streaming_callback) if self.closed(): - self._run_callback(callback, self._consume(self._read_buffer_size)) + if self._streaming_callback is not None: + self._run_callback(self._streaming_callback, + self._consume(self._read_buffer_size)) + self._run_callback(self._read_callback, + self._consume(self._read_buffer_size)) + self._streaming_callback = None + self._read_callback = None return self._read_until_close = True - self._read_callback = stack_context.wrap(callback) self._streaming_callback = stack_context.wrap(streaming_callback) - self._add_io_state(self.io_loop.READ) + self._try_inline_read() def write(self, data, callback=None): """Write the given data to this stream. - If callback is given, we call it when all of the buffered write + If ``callback`` is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. """ assert isinstance(data, bytes_type) self._check_closed() - self._write_buffer.append(data) + # We use bool(_write_buffer) as a proxy for write_buffer_size>0, + # so never put empty strings in the buffer. + if data: + # Break up large contiguous strings before inserting them in the + # write buffer, so we don't have to recopy the entire thing + # as we slice off pieces to send to the socket. + WRITE_BUFFER_CHUNK_SIZE = 128 * 1024 + if len(data) > WRITE_BUFFER_CHUNK_SIZE: + for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE): + self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE]) + else: + self._write_buffer.append(data) self._write_callback = stack_context.wrap(callback) - self._handle_write() - if self._write_buffer: - self._add_io_state(self.io_loop.WRITE) - self._maybe_add_error_listener() + if not self._connecting: + self._handle_write() + if self._write_buffer: + self._add_io_state(self.io_loop.WRITE) + self._maybe_add_error_listener() def set_close_callback(self, callback): """Call the given callback when the stream is closed.""" self._close_callback = stack_context.wrap(callback) - def close(self): - """Close this stream.""" - if self.socket is not None: + def close(self, exc_info=False): + """Close this stream. + + If ``exc_info`` is true, set the ``error`` attribute to the current + exception from `sys.exc_info` (or if ``exc_info`` is a tuple, + use that instead of `sys.exc_info`). + """ + if not self.closed(): + if exc_info: + if not isinstance(exc_info, tuple): + exc_info = sys.exc_info() + if any(exc_info): + self.error = exc_info[1] if self._read_until_close: + if (self._streaming_callback is not None and + self._read_buffer_size): + self._run_callback(self._streaming_callback, + self._consume(self._read_buffer_size)) callback = self._read_callback self._read_callback = None self._read_until_close = False self._run_callback(callback, self._consume(self._read_buffer_size)) if self._state is not None: - self.io_loop.remove_handler(self.socket.fileno()) + self.io_loop.remove_handler(self.fileno()) self._state = None - self.socket.close() - self.socket = None - if self._close_callback and self._pending_callbacks == 0: - # if there are pending callbacks, don't run the close callback - # until they're done (see _maybe_add_error_handler) - cb = self._close_callback - self._close_callback = None - self._run_callback(cb) + self.close_fd() + self._closed = True + self._maybe_run_close_callback() + + def _maybe_run_close_callback(self): + if (self.closed() and self._close_callback and + self._pending_callbacks == 0): + # if there are pending callbacks, don't run the close callback + # until they're done (see _maybe_add_error_handler) + cb = self._close_callback + self._close_callback = None + self._run_callback(cb) + # Delete any unfinished callbacks to break up reference cycles. + self._read_callback = self._write_callback = None def reading(self): """Returns true if we are currently reading from the stream.""" @@ -244,24 +277,40 @@ def closed(self): """Returns true if the stream has been closed.""" - return self.socket is None + return self._closed + + def set_nodelay(self, value): + """Sets the no-delay flag for this stream. + + By default, data written to TCP streams may be held for a time + to make the most efficient use of bandwidth (according to + Nagle's algorithm). The no-delay flag requests that data be + written as soon as possible, even if doing so would consume + additional bandwidth. + + This flag is currently defined only for TCP-based ``IOStreams``. + + .. versionadded:: 3.1 + """ + pass def _handle_events(self, fd, events): - if not self.socket: - logging.warning("Got events for closed stream %d", fd) + if self.closed(): + gen_log.warning("Got events for closed stream %d", fd) return try: if events & self.io_loop.READ: self._handle_read() - if not self.socket: + if self.closed(): return if events & self.io_loop.WRITE: if self._connecting: self._handle_connect() self._handle_write() - if not self.socket: + if self.closed(): return if events & self.io_loop.ERROR: + self.error = self.get_fd_error() # We may have queued up a user callback in _handle_read or # _handle_write, so don't close the IOStream until those # callbacks have had a chance to run. @@ -272,15 +321,17 @@ state |= self.io_loop.READ if self.writing(): state |= self.io_loop.WRITE + if state == self.io_loop.ERROR: + state |= self.io_loop.READ if state != self._state: assert self._state is not None, \ "shouldn't happen: _handle_events without self._state" self._state = state - self.io_loop.update_handler(self.socket.fileno(), self._state) + self.io_loop.update_handler(self.fileno(), self._state) except Exception: - logging.error("Uncaught exception, closing connection.", + gen_log.error("Uncaught exception, closing connection.", exc_info=True) - self.close() + self.close(exc_info=True) raise def _run_callback(self, callback, *args): @@ -289,13 +340,13 @@ try: callback(*args) except Exception: - logging.error("Uncaught exception, closing connection.", + app_log.error("Uncaught exception, closing connection.", exc_info=True) # Close the socket on an uncaught exception from a user callback # (It would eventually get closed when the socket object is # gc'd, but we don't want to rely on gc happening before we # run out of file descriptors) - self.close() + self.close(exc_info=True) # Re-raise the exception so that IOLoop.handle_callback_exception # can see it and log the error raise @@ -318,40 +369,72 @@ self.io_loop.add_callback(wrapper) def _handle_read(self): - while True: + try: try: - # Read from the socket until we get EWOULDBLOCK or equivalent. - # SSL sockets do some internal buffering, and if the data is - # sitting in the SSL object's buffer select() and friends - # can't see it; the only way to find out if it's there is to - # try to read it. - result = self._read_to_buffer() - except Exception: - self.close() - return - if result == 0: - break - else: - if self._read_from_buffer(): - return + # Pretend to have a pending callback so that an EOF in + # _read_to_buffer doesn't trigger an immediate close + # callback. At the end of this method we'll either + # estabilsh a real pending callback via + # _read_from_buffer or run the close callback. + # + # We need two try statements here so that + # pending_callbacks is decremented before the `except` + # clause below (which calls `close` and does need to + # trigger the callback) + self._pending_callbacks += 1 + while not self.closed(): + # Read from the socket until we get EWOULDBLOCK or equivalent. + # SSL sockets do some internal buffering, and if the data is + # sitting in the SSL object's buffer select() and friends + # can't see it; the only way to find out if it's there is to + # try to read it. + if self._read_to_buffer() == 0: + break + finally: + self._pending_callbacks -= 1 + except Exception: + gen_log.warning("error on read", exc_info=True) + self.close(exc_info=True) + return + if self._read_from_buffer(): + return + else: + self._maybe_run_close_callback() - def _read_from_socket(self): - """Attempts to read from the socket. + def _set_read_callback(self, callback): + assert not self._read_callback, "Already reading" + self._read_callback = stack_context.wrap(callback) + + def _try_inline_read(self): + """Attempt to complete the current read operation from buffered data. - Returns the data read or None if there is nothing to read. - May be overridden in subclasses. + If the read can be completed without blocking, schedules the + read callback on the next IOLoop iteration; otherwise starts + listening for reads on the socket. """ + # See if we've already got the data from a previous read + if self._read_from_buffer(): + return + self._check_closed() try: - chunk = self.socket.recv(self.read_chunk_size) - except socket.error, e: - if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): - return None - else: - raise - if not chunk: - self.close() - return None - return chunk + try: + # See comments in _handle_read about incrementing _pending_callbacks + self._pending_callbacks += 1 + while not self.closed(): + if self._read_to_buffer() == 0: + break + finally: + self._pending_callbacks -= 1 + except Exception: + # If there was an in _read_to_buffer, we called close() already, + # but couldn't run the close callback because of _pending_callbacks. + # Before we escape from this function, run the close callback if + # applicable. + self._maybe_run_close_callback() + raise + if self._read_from_buffer(): + return + self._maybe_add_error_listener() def _read_to_buffer(self): """Reads from the socket and appends the result to the read buffer. @@ -361,19 +444,23 @@ error closes the socket and raises an exception. """ try: - chunk = self._read_from_socket() - except socket.error, e: + chunk = self.read_from_fd() + except (socket.error, IOError, OSError) as e: # ssl.SSLError is a subclass of socket.error - logging.warning("Read error on %d: %s", - self.socket.fileno(), e) - self.close() + if e.args[0] == errno.ECONNRESET: + # Treat ECONNRESET as a connection close rather than + # an error to minimize log spam (the exception will + # be available on self.error for apps that care). + self.close(exc_info=True) + return + self.close(exc_info=True) raise if chunk is None: return 0 self._read_buffer.append(chunk) self._read_buffer_size += len(chunk) if self._read_buffer_size >= self.max_buffer_size: - logging.error("Reached maximum read buffer size") + gen_log.error("Reached maximum read buffer size") self.close() raise IOError("Reached maximum read buffer size") return len(chunk) @@ -383,65 +470,61 @@ Returns True if the read was completed. """ - if self._read_bytes is not None: - if self._streaming_callback is not None and self._read_buffer_size: - bytes_to_consume = min(self._read_bytes, self._read_buffer_size) + if self._streaming_callback is not None and self._read_buffer_size: + bytes_to_consume = self._read_buffer_size + if self._read_bytes is not None: + bytes_to_consume = min(self._read_bytes, bytes_to_consume) self._read_bytes -= bytes_to_consume - self._run_callback(self._streaming_callback, - self._consume(bytes_to_consume)) - if self._read_buffer_size >= self._read_bytes: - num_bytes = self._read_bytes - callback = self._read_callback - self._read_callback = None - self._streaming_callback = None - self._read_bytes = None - self._run_callback(callback, self._consume(num_bytes)) - return True + self._run_callback(self._streaming_callback, + self._consume(bytes_to_consume)) + if self._read_bytes is not None and self._read_buffer_size >= self._read_bytes: + num_bytes = self._read_bytes + callback = self._read_callback + self._read_callback = None + self._streaming_callback = None + self._read_bytes = None + self._run_callback(callback, self._consume(num_bytes)) + return True elif self._read_delimiter is not None: - _merge_prefix(self._read_buffer, sys.maxint) - loc = self._read_buffer[0].find(self._read_delimiter) - if loc != -1: - callback = self._read_callback - delimiter_len = len(self._read_delimiter) - self._read_callback = None - self._streaming_callback = None - self._read_delimiter = None - self._run_callback(callback, - self._consume(loc + delimiter_len)) - return True + # Multi-byte delimiters (e.g. '\r\n') may straddle two + # chunks in the read buffer, so we can't easily find them + # without collapsing the buffer. However, since protocols + # using delimited reads (as opposed to reads of a known + # length) tend to be "line" oriented, the delimiter is likely + # to be in the first few chunks. Merge the buffer gradually + # since large merges are relatively expensive and get undone in + # consume(). + if self._read_buffer: + while True: + loc = self._read_buffer[0].find(self._read_delimiter) + if loc != -1: + callback = self._read_callback + delimiter_len = len(self._read_delimiter) + self._read_callback = None + self._streaming_callback = None + self._read_delimiter = None + self._run_callback(callback, + self._consume(loc + delimiter_len)) + return True + if len(self._read_buffer) == 1: + break + _double_prefix(self._read_buffer) elif self._read_regex is not None: - _merge_prefix(self._read_buffer, sys.maxint) - m = self._read_regex.search(self._read_buffer[0]) - if m: - callback = self._read_callback - self._read_callback = None - self._streaming_callback = None - self._read_regex = None - self._run_callback(callback, self._consume(m.end())) - return True - elif self._read_until_close: - if self._streaming_callback is not None and self._read_buffer_size: - self._run_callback(self._streaming_callback, - self._consume(self._read_buffer_size)) + if self._read_buffer: + while True: + m = self._read_regex.search(self._read_buffer[0]) + if m is not None: + callback = self._read_callback + self._read_callback = None + self._streaming_callback = None + self._read_regex = None + self._run_callback(callback, self._consume(m.end())) + return True + if len(self._read_buffer) == 1: + break + _double_prefix(self._read_buffer) return False - def _handle_connect(self): - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - # IOLoop implementations may vary: some of them return - # an error state before the socket becomes writable, so - # in that case a connection failure would be handled by the - # error path in _handle_events instead of here. - logging.warning("Connect error on fd %d: %s", - self.socket.fileno(), errno.errorcode[err]) - self.close() - return - if self._connect_callback is not None: - callback = self._connect_callback - self._connect_callback = None - self._run_callback(callback) - self._connecting = False - def _handle_write(self): while self._write_buffer: try: @@ -452,7 +535,7 @@ # process. Therefore we must not call socket.send # with more than 128KB at a time. _merge_prefix(self._write_buffer, 128 * 1024) - num_bytes = self.socket.send(self._write_buffer[0]) + num_bytes = self.write_to_fd(self._write_buffer[0]) if num_bytes == 0: # With OpenSSL, if we couldn't write the entire buffer, # the very same string object must be used on the @@ -467,14 +550,18 @@ self._write_buffer_frozen = False _merge_prefix(self._write_buffer, num_bytes) self._write_buffer.popleft() - except socket.error, e: + except socket.error as e: if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): self._write_buffer_frozen = True break else: - logging.warning("Write error on %d: %s", - self.socket.fileno(), e) - self.close() + if e.args[0] not in (errno.EPIPE, errno.ECONNRESET): + # Broken pipe errors are usually caused by connection + # reset, and its better to not log EPIPE errors to + # minimize log spam + gen_log.warning("Write error on %d: %s", + self.fileno(), e) + self.close(exc_info=True) return if not self._write_buffer and self._write_callback: callback = self._write_callback @@ -483,24 +570,21 @@ def _consume(self, loc): if loc == 0: - return b("") + return b"" _merge_prefix(self._read_buffer, loc) self._read_buffer_size -= loc return self._read_buffer.popleft() def _check_closed(self): - if not self.socket: - raise IOError("Stream is closed") + if self.closed(): + raise StreamClosedError("Stream is closed") def _maybe_add_error_listener(self): if self._state is None and self._pending_callbacks == 0: - if self.socket is None: - cb = self._close_callback - if cb is not None: - self._close_callback = None - self._run_callback(cb) + if self.closed(): + self._maybe_run_close_callback() else: - self._add_io_state(0) + self._add_io_state(ioloop.IOLoop.READ) def _add_io_state(self, state): """Adds `state` (IOLoop.{READ,WRITE} flags) to our event handler. @@ -523,17 +607,162 @@ (since the write callback is optional so we can have a fast-path write with no `_run_callback`) """ - if self.socket is None: + if self.closed(): # connection has been closed, so there can be no future events return if self._state is None: self._state = ioloop.IOLoop.ERROR | state with stack_context.NullContext(): self.io_loop.add_handler( - self.socket.fileno(), self._handle_events, self._state) + self.fileno(), self._handle_events, self._state) elif not self._state & state: self._state = self._state | state - self.io_loop.update_handler(self.socket.fileno(), self._state) + self.io_loop.update_handler(self.fileno(), self._state) + + +class IOStream(BaseIOStream): + r"""Socket-based `IOStream` implementation. + + This class supports the read and write methods from `BaseIOStream` + plus a `connect` method. + + The ``socket`` parameter may either be connected or unconnected. + For server operations the socket is the result of calling + `socket.accept `. For client operations the + socket is created with `socket.socket`, and may either be + connected before passing it to the `IOStream` or connected with + `IOStream.connect`. + + A very simple (and broken) HTTP client using this class:: + + import tornado.ioloop + import tornado.iostream + import socket + + def send_request(): + stream.write(b"GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") + stream.read_until(b"\r\n\r\n", on_headers) + + def on_headers(data): + headers = {} + for line in data.split(b"\r\n"): + parts = line.split(b":") + if len(parts) == 2: + headers[parts[0].strip()] = parts[1].strip() + stream.read_bytes(int(headers[b"Content-Length"]), on_body) + + def on_body(data): + print data + stream.close() + tornado.ioloop.IOLoop.instance().stop() + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + stream = tornado.iostream.IOStream(s) + stream.connect(("friendfeed.com", 80), send_request) + tornado.ioloop.IOLoop.instance().start() + """ + def __init__(self, socket, *args, **kwargs): + self.socket = socket + self.socket.setblocking(False) + super(IOStream, self).__init__(*args, **kwargs) + + def fileno(self): + return self.socket.fileno() + + def close_fd(self): + self.socket.close() + self.socket = None + + def get_fd_error(self): + errno = self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_ERROR) + return socket.error(errno, os.strerror(errno)) + + def read_from_fd(self): + try: + chunk = self.socket.recv(self.read_chunk_size) + except socket.error as e: + if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + return None + else: + raise + if not chunk: + self.close() + return None + return chunk + + def write_to_fd(self, data): + return self.socket.send(data) + + def connect(self, address, callback=None, server_hostname=None): + """Connects the socket to a remote address without blocking. + + May only be called if the socket passed to the constructor was + not previously connected. The address parameter is in the + same format as for `socket.connect `, + i.e. a ``(host, port)`` tuple. If ``callback`` is specified, + it will be called when the connection is completed. + + If specified, the ``server_hostname`` parameter will be used + in SSL connections for certificate validation (if requested in + the ``ssl_options``) and SNI (if supported; requires + Python 3.2+). + + Note that it is safe to call `IOStream.write + ` while the connection is pending, in + which case the data will be written as soon as the connection + is ready. Calling `IOStream` read methods before the socket is + connected works on some platforms but is non-portable. + """ + self._connecting = True + try: + self.socket.connect(address) + except socket.error as e: + # In non-blocking mode we expect connect() to raise an + # exception with EINPROGRESS or EWOULDBLOCK. + # + # On freebsd, other errors such as ECONNREFUSED may be + # returned immediately when attempting to connect to + # localhost, so handle them the same way as an error + # reported later in _handle_connect. + if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): + gen_log.warning("Connect error on fd %d: %s", + self.socket.fileno(), e) + self.close(exc_info=True) + return + self._connect_callback = stack_context.wrap(callback) + self._add_io_state(self.io_loop.WRITE) + + def _handle_connect(self): + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + self.error = socket.error(err, os.strerror(err)) + # IOLoop implementations may vary: some of them return + # an error state before the socket becomes writable, so + # in that case a connection failure would be handled by the + # error path in _handle_events instead of here. + gen_log.warning("Connect error on fd %d: %s", + self.socket.fileno(), errno.errorcode[err]) + self.close() + return + if self._connect_callback is not None: + callback = self._connect_callback + self._connect_callback = None + self._run_callback(callback) + self._connecting = False + + def set_nodelay(self, value): + if (self.socket is not None and + self.socket.family in (socket.AF_INET, socket.AF_INET6)): + try: + self.socket.setsockopt(socket.IPPROTO_TCP, + socket.TCP_NODELAY, 1 if value else 0) + except socket.error as e: + # Sometimes setsockopt will fail if the socket is closed + # at the wrong time. This can happen with HTTPServer + # resetting the value to false between requests. + if e.errno != errno.EINVAL: + raise class SSLIOStream(IOStream): @@ -544,20 +773,21 @@ ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) - before constructing the SSLIOStream. Unconnected sockets will be - wrapped when IOStream.connect is finished. + before constructing the `SSLIOStream`. Unconnected sockets will be + wrapped when `IOStream.connect` is finished. """ def __init__(self, *args, **kwargs): - """Creates an SSLIOStream. - - If a dictionary is provided as keyword argument ssl_options, - it will be used as additional keyword arguments to ssl.wrap_socket. + """The ``ssl_options`` keyword argument may either be a dictionary + of keywords arguments for `ssl.wrap_socket`, or an `ssl.SSLContext` + object. """ self._ssl_options = kwargs.pop('ssl_options', {}) super(SSLIOStream, self).__init__(*args, **kwargs) self._ssl_accepting = True self._handshake_reading = False self._handshake_writing = False + self._ssl_connect_callback = None + self._server_hostname = None def reading(self): return self._handshake_reading or super(SSLIOStream, self).reading() @@ -571,7 +801,7 @@ self._handshake_reading = False self._handshake_writing = False self.socket.do_handshake() - except ssl.SSLError, err: + except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_WANT_READ: self._handshake_reading = True return @@ -580,17 +810,60 @@ return elif err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): - return self.close() + return self.close(exc_info=True) elif err.args[0] == ssl.SSL_ERROR_SSL: - logging.warning("SSL Error on %d: %s", self.socket.fileno(), err) - return self.close() + try: + peer = self.socket.getpeername() + except Exception: + peer = '(not connected)' + gen_log.warning("SSL Error on %d %s: %s", + self.socket.fileno(), peer, err) + return self.close(exc_info=True) raise - except socket.error, err: - if err.args[0] == errno.ECONNABORTED: - return self.close() + except socket.error as err: + if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET): + return self.close(exc_info=True) + except AttributeError: + # On Linux, if the connection was reset before the call to + # wrap_socket, do_handshake will fail with an + # AttributeError. + return self.close(exc_info=True) else: self._ssl_accepting = False - super(SSLIOStream, self)._handle_connect() + if not self._verify_cert(self.socket.getpeercert()): + self.close() + return + if self._ssl_connect_callback is not None: + callback = self._ssl_connect_callback + self._ssl_connect_callback = None + self._run_callback(callback) + + def _verify_cert(self, peercert): + """Returns True if peercert is valid according to the configured + validation mode and hostname. + + The ssl handshake already tested the certificate for a valid + CA signature; the only thing that remains is to check + the hostname. + """ + if isinstance(self._ssl_options, dict): + verify_mode = self._ssl_options.get('cert_reqs', ssl.CERT_NONE) + elif isinstance(self._ssl_options, ssl.SSLContext): + verify_mode = self._ssl_options.verify_mode + assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL) + if verify_mode == ssl.CERT_NONE or self._server_hostname is None: + return True + cert = self.socket.getpeercert() + if cert is None and verify_mode == ssl.CERT_REQUIRED: + gen_log.warning("No SSL certificate given") + return False + try: + ssl_match_hostname(peercert, self._server_hostname) + except SSLCertificateError: + gen_log.warning("Invalid SSL certificate", exc_info=True) + return False + else: + return True def _handle_read(self): if self._ssl_accepting: @@ -604,17 +877,31 @@ return super(SSLIOStream, self)._handle_write() - def _handle_connect(self): - self.socket = ssl.wrap_socket(self.socket, - do_handshake_on_connect=False, - **self._ssl_options) - # Don't call the superclass's _handle_connect (which is responsible - # for telling the application that the connection is complete) - # until we've completed the SSL handshake (so certificates are - # available, etc). + def connect(self, address, callback=None, server_hostname=None): + # Save the user's callback and run it after the ssl handshake + # has completed. + self._ssl_connect_callback = stack_context.wrap(callback) + self._server_hostname = server_hostname + super(SSLIOStream, self).connect(address, callback=None) + def _handle_connect(self): + # When the connection is complete, wrap the socket for SSL + # traffic. Note that we do this by overriding _handle_connect + # instead of by passing a callback to super().connect because + # user callbacks are enqueued asynchronously on the IOLoop, + # but since _handle_events calls _handle_connect immediately + # followed by _handle_write we need this to be synchronous. + self.socket = ssl_wrap_socket(self.socket, self._ssl_options, + server_hostname=self._server_hostname, + do_handshake_on_connect=False) + super(SSLIOStream, self)._handle_connect() - def _read_from_socket(self): + def read_from_fd(self): + if self._ssl_accepting: + # If the handshake hasn't finished yet, there can't be anything + # to read (attempting to read may or may not raise an exception + # depending on the SSL version) + return None try: # SSLSocket objects have both a read() and recv() method, # while regular sockets only have recv(). @@ -622,14 +909,14 @@ # called when there is nothing to read, so we have to use # read() instead. chunk = self.socket.read(self.read_chunk_size) - except ssl.SSLError, e: + except ssl.SSLError as e: # SSLError is a subclass of socket.error, so this except # block must come first. if e.args[0] == ssl.SSL_ERROR_WANT_READ: return None else: raise - except socket.error, e: + except socket.error as e: if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): return None else: @@ -639,22 +926,73 @@ return None return chunk + +class PipeIOStream(BaseIOStream): + """Pipe-based `IOStream` implementation. + + The constructor takes an integer file descriptor (such as one returned + by `os.pipe`) rather than an open file object. Pipes are generally + one-way, so a `PipeIOStream` can be used for reading or writing but not + both. + """ + def __init__(self, fd, *args, **kwargs): + self.fd = fd + _set_nonblocking(fd) + super(PipeIOStream, self).__init__(*args, **kwargs) + + def fileno(self): + return self.fd + + def close_fd(self): + os.close(self.fd) + + def write_to_fd(self, data): + return os.write(self.fd, data) + + def read_from_fd(self): + try: + chunk = os.read(self.fd, self.read_chunk_size) + except (IOError, OSError) as e: + if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + return None + elif e.args[0] == errno.EBADF: + # If the writing half of a pipe is closed, select will + # report it as readable but reads will fail with EBADF. + self.close(exc_info=True) + return None + else: + raise + if not chunk: + self.close() + return None + return chunk + + +def _double_prefix(deque): + """Grow by doubling, but don't split the second chunk just because the + first one is small. + """ + new_len = max(len(deque[0]) * 2, + (len(deque[0]) + len(deque[1]))) + _merge_prefix(deque, new_len) + + def _merge_prefix(deque, size): """Replace the first entries in a deque of strings with a single string of up to size bytes. >>> d = collections.deque(['abc', 'de', 'fghi', 'j']) - >>> _merge_prefix(d, 5); print d + >>> _merge_prefix(d, 5); print(d) deque(['abcde', 'fghi', 'j']) Strings will be split as necessary to reach the desired size. - >>> _merge_prefix(d, 7); print d + >>> _merge_prefix(d, 7); print(d) deque(['abcdefg', 'hi', 'j']) - >>> _merge_prefix(d, 3); print d + >>> _merge_prefix(d, 3); print(d) deque(['abc', 'defg', 'hi', 'j']) - >>> _merge_prefix(d, 100); print d + >>> _merge_prefix(d, 100); print(d) deque(['abcdefghij']) """ if len(deque) == 1 and len(deque[0]) <= size: @@ -674,7 +1012,8 @@ if prefix: deque.appendleft(type(prefix[0])().join(prefix)) if not deque: - deque.appendleft(b("")) + deque.appendleft(b"") + def doctests(): import doctest diff -Nru python-tornado-2.1.0/tornado/locale.py python-tornado-3.1.1/tornado/locale.py --- python-tornado-2.1.0/tornado/locale.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/locale.py 2013-08-04 19:34:21.000000000 +0000 @@ -1,5 +1,5 @@ #!/usr/bin/env python -# +# -*- coding: utf-8 -*- # Copyright 2009 Facebook # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -18,38 +18,45 @@ To load a locale and generate a translated string:: - user_locale = locale.get("es_LA") + user_locale = tornado.locale.get("es_LA") print user_locale.translate("Sign out") -locale.get() returns the closest matching locale, not necessarily the +`tornado.locale.get()` returns the closest matching locale, not necessarily the specific locale you requested. You can support pluralization with -additional arguments to translate(), e.g.:: +additional arguments to `~Locale.translate()`, e.g.:: people = [...] message = user_locale.translate( "%(list)s is online", "%(list)s are online", len(people)) print message % {"list": user_locale.list(people)} -The first string is chosen if len(people) == 1, otherwise the second +The first string is chosen if ``len(people) == 1``, otherwise the second string is chosen. -Applications should call one of load_translations (which uses a simple -CSV format) or load_gettext_translations (which uses the .mo format -supported by gettext and related tools). If neither method is called, -the locale.translate method will simply return the original string. +Applications should call one of `load_translations` (which uses a simple +CSV format) or `load_gettext_translations` (which uses the ``.mo`` format +supported by `gettext` and related tools). If neither method is called, +the `Locale.translate` method will simply return the original string. """ +from __future__ import absolute_import, division, print_function, with_statement + import csv import datetime -import logging +import numbers import os import re +from tornado import escape +from tornado.log import gen_log +from tornado.util import u + _default_locale = "en_US" _translations = {} _supported_locales = frozenset([_default_locale]) _use_gettext = False + def get(*locale_codes): """Returns the closest match for the given locale codes. @@ -57,15 +64,15 @@ or a loose match for the code (e.g., "en" for "en_US"), we return the locale. Otherwise we move to the next code in the list. - By default we return en_US if no translations are found for any of + By default we return ``en_US`` if no translations are found for any of the specified locales. You can change the default locale with - set_default_locale() below. + `set_default_locale()`. """ return Locale.get_closest(*locale_codes) def set_default_locale(code): - """Sets the default locale, used in get_closest_locale(). + """Sets the default locale. The default locale is assumed to be the language used for all strings in the system. The translations loaded from disk are mappings from @@ -75,82 +82,95 @@ global _default_locale global _supported_locales _default_locale = code - _supported_locales = frozenset(_translations.keys() + [_default_locale]) + _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) def load_translations(directory): - u"""Loads translations from CSV files in a directory. + """Loads translations from CSV files in a directory. Translations are strings with optional Python-style named placeholders - (e.g., "My name is %(name)s") and their associated translations. + (e.g., ``My name is %(name)s``) and their associated translations. - The directory should have translation files of the form LOCALE.csv, - e.g. es_GT.csv. The CSV files should have two or three columns: string, + The directory should have translation files of the form ``LOCALE.csv``, + e.g. ``es_GT.csv``. The CSV files should have two or three columns: string, translation, and an optional plural indicator. Plural indicators should be one of "plural" or "singular". A given string can have both singular - and plural forms. For example "%(name)s liked this" may have a + and plural forms. For example ``%(name)s liked this`` may have a different verb conjugation depending on whether %(name)s is one name or a list of names. There should be two rows in the CSV file for that string, one with plural indicator "singular", and one "plural". For strings with no verbs that would change on translation, simply use "unknown" or the empty string (or don't include the column at all). - The file is read using the csv module in the default "excel" dialect. + The file is read using the `csv` module in the default "excel" dialect. In this format there should not be spaces after the commas. - Example translation es_LA.csv: + Example translation ``es_LA.csv``:: "I love you","Te amo" - "%(name)s liked this","A %(name)s les gust\u00f3 esto","plural" - "%(name)s liked this","A %(name)s le gust\u00f3 esto","singular" + "%(name)s liked this","A %(name)s les gustó esto","plural" + "%(name)s liked this","A %(name)s le gustó esto","singular" """ global _translations global _supported_locales _translations = {} for path in os.listdir(directory): - if not path.endswith(".csv"): continue + if not path.endswith(".csv"): + continue locale, extension = path.split(".") if not re.match("[a-z]+(_[A-Z]+)?$", locale): - logging.error("Unrecognized locale %r (path: %s)", locale, + gen_log.error("Unrecognized locale %r (path: %s)", locale, os.path.join(directory, path)) continue - f = open(os.path.join(directory, path), "r") + full_path = os.path.join(directory, path) + try: + # python 3: csv.reader requires a file open in text mode. + # Force utf8 to avoid dependence on $LANG environment variable. + f = open(full_path, "r", encoding="utf-8") + except TypeError: + # python 2: files return byte strings, which are decoded below. + f = open(full_path, "r") _translations[locale] = {} for i, row in enumerate(csv.reader(f)): - if not row or len(row) < 2: continue - row = [c.decode("utf-8").strip() for c in row] + if not row or len(row) < 2: + continue + row = [escape.to_unicode(c).strip() for c in row] english, translation = row[:2] if len(row) > 2: plural = row[2] or "unknown" else: plural = "unknown" if plural not in ("plural", "singular", "unknown"): - logging.error("Unrecognized plural indicator %r in %s line %d", + gen_log.error("Unrecognized plural indicator %r in %s line %d", plural, path, i + 1) continue _translations[locale].setdefault(plural, {})[english] = translation f.close() - _supported_locales = frozenset(_translations.keys() + [_default_locale]) - logging.info("Supported locales: %s", sorted(_supported_locales)) + _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) + gen_log.debug("Supported locales: %s", sorted(_supported_locales)) + def load_gettext_translations(directory, domain): - """Loads translations from gettext's locale tree + """Loads translations from `gettext`'s locale tree - Locale tree is similar to system's /usr/share/locale, like: + Locale tree is similar to system's ``/usr/share/locale``, like:: - {directory}/{lang}/LC_MESSAGES/{domain}.mo + {directory}/{lang}/LC_MESSAGES/{domain}.mo Three steps are required to have you app translated: - 1. Generate POT translation file - xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc + 1. Generate POT translation file:: + + xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc - 2. Merge against existing POT file: - msgmerge old.po cyclone.po > new.po + 2. Merge against existing POT file:: - 3. Compile: - msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo + msgmerge old.po mydomain.po > new.po + + 3. Compile:: + + msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo """ import gettext global _translations @@ -158,21 +178,23 @@ global _use_gettext _translations = {} for lang in os.listdir(directory): - if lang.startswith('.'): continue # skip .svn, etc - if os.path.isfile(os.path.join(directory, lang)): continue + if lang.startswith('.'): + continue # skip .svn, etc + if os.path.isfile(os.path.join(directory, lang)): + continue try: - os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain+".mo")) + os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo")) _translations[lang] = gettext.translation(domain, directory, languages=[lang]) - except Exception, e: - logging.error("Cannot load translation for '%s': %s", lang, str(e)) + except Exception as e: + gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) continue - _supported_locales = frozenset(_translations.keys() + [_default_locale]) + _supported_locales = frozenset(list(_translations.keys()) + [_default_locale]) _use_gettext = True - logging.info("Supported locales: %s", sorted(_supported_locales)) + gen_log.debug("Supported locales: %s", sorted(_supported_locales)) -def get_supported_locales(cls): +def get_supported_locales(): """Returns a list of all the supported locale codes.""" return _supported_locales @@ -187,7 +209,8 @@ def get_closest(cls, *locale_codes): """Returns the closest match for the given locale code.""" for code in locale_codes: - if not code: continue + if not code: + continue code = code.replace("-", "_") parts = code.split("_") if len(parts) > 2: @@ -222,7 +245,7 @@ def __init__(self, code, translations): self.code = code - self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown") + self.name = LOCALE_NAMES.get(code, {}).get("name", u("Unknown")) self.rtl = False for prefix in ["fa", "ar", "he"]: if self.code.startswith(prefix): @@ -243,9 +266,10 @@ def translate(self, message, plural_message=None, count=None): """Returns the translation for the given message for this locale. - If plural_message is given, you must also provide count. We return - plural_message when count != 1, and we return the singular form - for the given message when count == 1. + If ``plural_message`` is given, you must also provide + ``count``. We return ``plural_message`` when ``count != 1``, + and we return the singular form for the given message when + ``count == 1``. """ raise NotImplementedError() @@ -254,17 +278,17 @@ """Formats the given date (which should be GMT). By default, we return a relative time (e.g., "2 minutes ago"). You - can return an absolute date string with relative=False. + can return an absolute date string with ``relative=False``. You can force a full format date ("July 10, 1980") with - full_format=True. + ``full_format=True``. This method is primarily intended for dates in the past. For dates in the future, we fall back to full format. """ if self.code.startswith("ru"): relative = False - if type(date) in (int, long, float): + if isinstance(date, numbers.Real): date = datetime.datetime.utcfromtimestamp(date) now = datetime.datetime.utcnow() if date > now: @@ -289,40 +313,40 @@ if relative and days == 0: if seconds < 50: return _("1 second ago", "%(seconds)d seconds ago", - seconds) % { "seconds": seconds } + seconds) % {"seconds": seconds} if seconds < 50 * 60: minutes = round(seconds / 60.0) return _("1 minute ago", "%(minutes)d minutes ago", - minutes) % { "minutes": minutes } + minutes) % {"minutes": minutes} hours = round(seconds / (60.0 * 60)) return _("1 hour ago", "%(hours)d hours ago", - hours) % { "hours": hours } + hours) % {"hours": hours} if days == 0: format = _("%(time)s") elif days == 1 and local_date.day == local_yesterday.day and \ - relative: + relative: format = _("yesterday") if shorter else \ - _("yesterday at %(time)s") + _("yesterday at %(time)s") elif days < 5: format = _("%(weekday)s") if shorter else \ - _("%(weekday)s at %(time)s") + _("%(weekday)s at %(time)s") elif days < 334: # 11mo, since confusing for same month last year format = _("%(month_name)s %(day)s") if shorter else \ - _("%(month_name)s %(day)s at %(time)s") + _("%(month_name)s %(day)s at %(time)s") if format is None: format = _("%(month_name)s %(day)s, %(year)s") if shorter else \ - _("%(month_name)s %(day)s, %(year)s at %(time)s") + _("%(month_name)s %(day)s, %(year)s at %(time)s") tfhour_clock = self.code not in ("en", "en_US", "zh_CN") if tfhour_clock: str_time = "%d:%02d" % (local_date.hour, local_date.minute) elif self.code == "zh_CN": str_time = "%s%d:%02d" % ( - (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12], + (u('\u4e0a\u5348'), u('\u4e0b\u5348'))[local_date.hour >= 12], local_date.hour % 12 or 12, local_date.minute) else: str_time = "%d:%02d %s" % ( @@ -341,7 +365,7 @@ """Formats the given date as a day of week. Example: "Monday, January 22". You can remove the day of week with - dow=False. + ``dow=False``. """ local_date = date - datetime.timedelta(minutes=gmt_offset) _ = self.translate @@ -364,9 +388,11 @@ of size 1. """ _ = self.translate - if len(parts) == 0: return "" - if len(parts) == 1: return parts[0] - comma = u' \u0648 ' if self.code.startswith("fa") else u", " + if len(parts) == 0: + return "" + if len(parts) == 1: + return parts[0] + comma = u(' \u0648 ') if self.code.startswith("fa") else u(", ") return _("%(commas)s and %(last)s") % { "commas": comma.join(parts[:-1]), "last": parts[len(parts) - 1], @@ -383,6 +409,7 @@ value = value[:-3] return ",".join(reversed(parts)) + class CSVLocale(Locale): """Locale implementation using tornado's CSV translation format.""" def translate(self, message, plural_message=None, count=None): @@ -397,76 +424,90 @@ message_dict = self.translations.get("unknown", {}) return message_dict.get(message, message) + class GettextLocale(Locale): - """Locale implementation using the gettext module.""" + """Locale implementation using the `gettext` module.""" + def __init__(self, code, translations): + try: + # python 2 + self.ngettext = translations.ungettext + self.gettext = translations.ugettext + except AttributeError: + # python 3 + self.ngettext = translations.ngettext + self.gettext = translations.gettext + # self.gettext must exist before __init__ is called, since it + # calls into self.translate + super(GettextLocale, self).__init__(code, translations) + def translate(self, message, plural_message=None, count=None): if plural_message is not None: assert count is not None - return self.translations.ungettext(message, plural_message, count) + return self.ngettext(message, plural_message, count) else: - return self.translations.ugettext(message) + return self.gettext(message) LOCALE_NAMES = { - "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, - "am_ET": {"name_en": u"Amharic", "name": u'\u12a0\u121b\u122d\u129b'}, - "ar_AR": {"name_en": u"Arabic", "name": u"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"}, - "bg_BG": {"name_en": u"Bulgarian", "name": u"\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"}, - "bn_IN": {"name_en": u"Bengali", "name": u"\u09ac\u09be\u0982\u09b2\u09be"}, - "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"}, - "ca_ES": {"name_en": u"Catalan", "name": u"Catal\xe0"}, - "cs_CZ": {"name_en": u"Czech", "name": u"\u010ce\u0161tina"}, - "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"}, - "da_DK": {"name_en": u"Danish", "name": u"Dansk"}, - "de_DE": {"name_en": u"German", "name": u"Deutsch"}, - "el_GR": {"name_en": u"Greek", "name": u"\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"}, - "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"}, - "en_US": {"name_en": u"English (US)", "name": u"English (US)"}, - "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Espa\xf1ol (Espa\xf1a)"}, - "es_LA": {"name_en": u"Spanish", "name": u"Espa\xf1ol"}, - "et_EE": {"name_en": u"Estonian", "name": u"Eesti"}, - "eu_ES": {"name_en": u"Basque", "name": u"Euskara"}, - "fa_IR": {"name_en": u"Persian", "name": u"\u0641\u0627\u0631\u0633\u06cc"}, - "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"}, - "fr_CA": {"name_en": u"French (Canada)", "name": u"Fran\xe7ais (Canada)"}, - "fr_FR": {"name_en": u"French", "name": u"Fran\xe7ais"}, - "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"}, - "gl_ES": {"name_en": u"Galician", "name": u"Galego"}, - "he_IL": {"name_en": u"Hebrew", "name": u"\u05e2\u05d1\u05e8\u05d9\u05ea"}, - "hi_IN": {"name_en": u"Hindi", "name": u"\u0939\u093f\u0928\u094d\u0926\u0940"}, - "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"}, - "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"}, - "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"}, - "is_IS": {"name_en": u"Icelandic", "name": u"\xcdslenska"}, - "it_IT": {"name_en": u"Italian", "name": u"Italiano"}, - "ja_JP": {"name_en": u"Japanese", "name": u"\u65e5\u672c\u8a9e"}, - "ko_KR": {"name_en": u"Korean", "name": u"\ud55c\uad6d\uc5b4"}, - "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvi\u0173"}, - "lv_LV": {"name_en": u"Latvian", "name": u"Latvie\u0161u"}, - "mk_MK": {"name_en": u"Macedonian", "name": u"\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438"}, - "ml_IN": {"name_en": u"Malayalam", "name": u"\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02"}, - "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"}, - "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokm\xe5l)"}, - "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"}, - "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"}, - "pa_IN": {"name_en": u"Punjabi", "name": u"\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40"}, - "pl_PL": {"name_en": u"Polish", "name": u"Polski"}, - "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Portugu\xeas (Brasil)"}, - "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Portugu\xeas (Portugal)"}, - "ro_RO": {"name_en": u"Romanian", "name": u"Rom\xe2n\u0103"}, - "ru_RU": {"name_en": u"Russian", "name": u"\u0420\u0443\u0441\u0441\u043a\u0438\u0439"}, - "sk_SK": {"name_en": u"Slovak", "name": u"Sloven\u010dina"}, - "sl_SI": {"name_en": u"Slovenian", "name": u"Sloven\u0161\u010dina"}, - "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"}, - "sr_RS": {"name_en": u"Serbian", "name": u"\u0421\u0440\u043f\u0441\u043a\u0438"}, - "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"}, - "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"}, - "ta_IN": {"name_en": u"Tamil", "name": u"\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"}, - "te_IN": {"name_en": u"Telugu", "name": u"\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41"}, - "th_TH": {"name_en": u"Thai", "name": u"\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"}, - "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"}, - "tr_TR": {"name_en": u"Turkish", "name": u"T\xfcrk\xe7e"}, - "uk_UA": {"name_en": u"Ukraini ", "name": u"\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"}, - "vi_VN": {"name_en": u"Vietnamese", "name": u"Ti\u1ebfng Vi\u1ec7t"}, - "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"\u4e2d\u6587(\u7b80\u4f53)"}, - "zh_TW": {"name_en": u"Chinese (Traditional)", "name": u"\u4e2d\u6587(\u7e41\u9ad4)"}, + "af_ZA": {"name_en": u("Afrikaans"), "name": u("Afrikaans")}, + "am_ET": {"name_en": u("Amharic"), "name": u('\u12a0\u121b\u122d\u129b')}, + "ar_AR": {"name_en": u("Arabic"), "name": u("\u0627\u0644\u0639\u0631\u0628\u064a\u0629")}, + "bg_BG": {"name_en": u("Bulgarian"), "name": u("\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438")}, + "bn_IN": {"name_en": u("Bengali"), "name": u("\u09ac\u09be\u0982\u09b2\u09be")}, + "bs_BA": {"name_en": u("Bosnian"), "name": u("Bosanski")}, + "ca_ES": {"name_en": u("Catalan"), "name": u("Catal\xe0")}, + "cs_CZ": {"name_en": u("Czech"), "name": u("\u010ce\u0161tina")}, + "cy_GB": {"name_en": u("Welsh"), "name": u("Cymraeg")}, + "da_DK": {"name_en": u("Danish"), "name": u("Dansk")}, + "de_DE": {"name_en": u("German"), "name": u("Deutsch")}, + "el_GR": {"name_en": u("Greek"), "name": u("\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac")}, + "en_GB": {"name_en": u("English (UK)"), "name": u("English (UK)")}, + "en_US": {"name_en": u("English (US)"), "name": u("English (US)")}, + "es_ES": {"name_en": u("Spanish (Spain)"), "name": u("Espa\xf1ol (Espa\xf1a)")}, + "es_LA": {"name_en": u("Spanish"), "name": u("Espa\xf1ol")}, + "et_EE": {"name_en": u("Estonian"), "name": u("Eesti")}, + "eu_ES": {"name_en": u("Basque"), "name": u("Euskara")}, + "fa_IR": {"name_en": u("Persian"), "name": u("\u0641\u0627\u0631\u0633\u06cc")}, + "fi_FI": {"name_en": u("Finnish"), "name": u("Suomi")}, + "fr_CA": {"name_en": u("French (Canada)"), "name": u("Fran\xe7ais (Canada)")}, + "fr_FR": {"name_en": u("French"), "name": u("Fran\xe7ais")}, + "ga_IE": {"name_en": u("Irish"), "name": u("Gaeilge")}, + "gl_ES": {"name_en": u("Galician"), "name": u("Galego")}, + "he_IL": {"name_en": u("Hebrew"), "name": u("\u05e2\u05d1\u05e8\u05d9\u05ea")}, + "hi_IN": {"name_en": u("Hindi"), "name": u("\u0939\u093f\u0928\u094d\u0926\u0940")}, + "hr_HR": {"name_en": u("Croatian"), "name": u("Hrvatski")}, + "hu_HU": {"name_en": u("Hungarian"), "name": u("Magyar")}, + "id_ID": {"name_en": u("Indonesian"), "name": u("Bahasa Indonesia")}, + "is_IS": {"name_en": u("Icelandic"), "name": u("\xcdslenska")}, + "it_IT": {"name_en": u("Italian"), "name": u("Italiano")}, + "ja_JP": {"name_en": u("Japanese"), "name": u("\u65e5\u672c\u8a9e")}, + "ko_KR": {"name_en": u("Korean"), "name": u("\ud55c\uad6d\uc5b4")}, + "lt_LT": {"name_en": u("Lithuanian"), "name": u("Lietuvi\u0173")}, + "lv_LV": {"name_en": u("Latvian"), "name": u("Latvie\u0161u")}, + "mk_MK": {"name_en": u("Macedonian"), "name": u("\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438")}, + "ml_IN": {"name_en": u("Malayalam"), "name": u("\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02")}, + "ms_MY": {"name_en": u("Malay"), "name": u("Bahasa Melayu")}, + "nb_NO": {"name_en": u("Norwegian (bokmal)"), "name": u("Norsk (bokm\xe5l)")}, + "nl_NL": {"name_en": u("Dutch"), "name": u("Nederlands")}, + "nn_NO": {"name_en": u("Norwegian (nynorsk)"), "name": u("Norsk (nynorsk)")}, + "pa_IN": {"name_en": u("Punjabi"), "name": u("\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40")}, + "pl_PL": {"name_en": u("Polish"), "name": u("Polski")}, + "pt_BR": {"name_en": u("Portuguese (Brazil)"), "name": u("Portugu\xeas (Brasil)")}, + "pt_PT": {"name_en": u("Portuguese (Portugal)"), "name": u("Portugu\xeas (Portugal)")}, + "ro_RO": {"name_en": u("Romanian"), "name": u("Rom\xe2n\u0103")}, + "ru_RU": {"name_en": u("Russian"), "name": u("\u0420\u0443\u0441\u0441\u043a\u0438\u0439")}, + "sk_SK": {"name_en": u("Slovak"), "name": u("Sloven\u010dina")}, + "sl_SI": {"name_en": u("Slovenian"), "name": u("Sloven\u0161\u010dina")}, + "sq_AL": {"name_en": u("Albanian"), "name": u("Shqip")}, + "sr_RS": {"name_en": u("Serbian"), "name": u("\u0421\u0440\u043f\u0441\u043a\u0438")}, + "sv_SE": {"name_en": u("Swedish"), "name": u("Svenska")}, + "sw_KE": {"name_en": u("Swahili"), "name": u("Kiswahili")}, + "ta_IN": {"name_en": u("Tamil"), "name": u("\u0ba4\u0bae\u0bbf\u0bb4\u0bcd")}, + "te_IN": {"name_en": u("Telugu"), "name": u("\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41")}, + "th_TH": {"name_en": u("Thai"), "name": u("\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22")}, + "tl_PH": {"name_en": u("Filipino"), "name": u("Filipino")}, + "tr_TR": {"name_en": u("Turkish"), "name": u("T\xfcrk\xe7e")}, + "uk_UA": {"name_en": u("Ukraini "), "name": u("\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430")}, + "vi_VN": {"name_en": u("Vietnamese"), "name": u("Ti\u1ebfng Vi\u1ec7t")}, + "zh_CN": {"name_en": u("Chinese (Simplified)"), "name": u("\u4e2d\u6587(\u7b80\u4f53)")}, + "zh_TW": {"name_en": u("Chinese (Traditional)"), "name": u("\u4e2d\u6587(\u7e41\u9ad4)")}, } diff -Nru python-tornado-2.1.0/tornado/log.py python-tornado-3.1.1/tornado/log.py --- python-tornado-2.1.0/tornado/log.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/log.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Logging support for Tornado. + +Tornado uses three logger streams: + +* ``tornado.access``: Per-request logging for Tornado's HTTP servers (and + potentially other servers in the future) +* ``tornado.application``: Logging of errors from application code (i.e. + uncaught exceptions from callbacks) +* ``tornado.general``: General-purpose logging, including any errors + or warnings from Tornado itself. + +These streams may be configured independently using the standard library's +`logging` module. For example, you may wish to send ``tornado.access`` logs +to a separate file for analysis. +""" +from __future__ import absolute_import, division, print_function, with_statement + +import logging +import logging.handlers +import sys +import time + +from tornado.escape import _unicode +from tornado.util import unicode_type, basestring_type + +try: + import curses +except ImportError: + curses = None + +# Logger objects for internal tornado use +access_log = logging.getLogger("tornado.access") +app_log = logging.getLogger("tornado.application") +gen_log = logging.getLogger("tornado.general") + + +def _stderr_supports_color(): + color = False + if curses and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + + +class LogFormatter(logging.Formatter): + """Log formatter used in Tornado. + + Key features of this formatter are: + + * Color support when logging to a terminal that supports it. + * Timestamps on every log line. + * Robust against str/bytes encoding problems. + + This formatter is enabled automatically by + `tornado.options.parse_command_line` (unless ``--logging=none`` is + used). + """ + def __init__(self, color=True, *args, **kwargs): + logging.Formatter.__init__(self, *args, **kwargs) + self._color = color and _stderr_supports_color() + if self._color: + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = unicode_type(fg_color, "ascii") + self._colors = { + logging.DEBUG: unicode_type(curses.tparm(fg_color, 4), # Blue + "ascii"), + logging.INFO: unicode_type(curses.tparm(fg_color, 2), # Green + "ascii"), + logging.WARNING: unicode_type(curses.tparm(fg_color, 3), # Yellow + "ascii"), + logging.ERROR: unicode_type(curses.tparm(fg_color, 1), # Red + "ascii"), + } + self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") + + def format(self, record): + try: + record.message = record.getMessage() + except Exception as e: + record.message = "Bad message (%r): %r" % (e, record.__dict__) + assert isinstance(record.message, basestring_type) # guaranteed by logging + record.asctime = time.strftime( + "%y%m%d %H:%M:%S", self.converter(record.created)) + prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ + record.__dict__ + if self._color: + prefix = (self._colors.get(record.levelno, self._normal) + + prefix + self._normal) + + # Encoding notes: The logging module prefers to work with character + # strings, but only enforces that log messages are instances of + # basestring. In python 2, non-ascii bytestrings will make + # their way through the logging framework until they blow up with + # an unhelpful decoding error (with this formatter it happens + # when we attach the prefix, but there are other opportunities for + # exceptions further along in the framework). + # + # If a byte string makes it this far, convert it to unicode to + # ensure it will make it out to the logs. Use repr() as a fallback + # to ensure that all byte strings can be converted successfully, + # but don't do it by default so we don't add extra quotes to ascii + # bytestrings. This is a bit of a hacky place to do this, but + # it's worth it since the encoding errors that would otherwise + # result are so useless (and tornado is fond of using utf8-encoded + # byte strings whereever possible). + def safe_unicode(s): + try: + return _unicode(s) + except UnicodeDecodeError: + return repr(s) + + formatted = prefix + " " + safe_unicode(record.message) + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + # exc_text contains multiple lines. We need to safe_unicode + # each line separately so that non-utf8 bytes don't cause + # all the newlines to turn into '\n'. + lines = [formatted.rstrip()] + lines.extend(safe_unicode(ln) for ln in record.exc_text.split('\n')) + formatted = '\n'.join(lines) + return formatted.replace("\n", "\n ") + + +def enable_pretty_logging(options=None, logger=None): + """Turns on formatted logging output as configured. + + This is called automaticaly by `tornado.options.parse_command_line` + and `tornado.options.parse_config_file`. + """ + if options is None: + from tornado.options import options + if options.logging == 'none': + return + if logger is None: + logger = logging.getLogger() + logger.setLevel(getattr(logging, options.logging.upper())) + if options.log_file_prefix: + channel = logging.handlers.RotatingFileHandler( + filename=options.log_file_prefix, + maxBytes=options.log_file_max_size, + backupCount=options.log_file_num_backups) + channel.setFormatter(LogFormatter(color=False)) + logger.addHandler(channel) + + if (options.log_to_stderr or + (options.log_to_stderr is None and not logger.handlers)): + # Set up color if we are in a tty and curses is installed + channel = logging.StreamHandler() + channel.setFormatter(LogFormatter()) + logger.addHandler(channel) + + +def define_logging_options(options=None): + if options is None: + # late import to prevent cycle + from tornado.options import options + options.define("logging", default="info", + help=("Set the Python log level. If 'none', tornado won't touch the " + "logging configuration."), + metavar="debug|info|warning|error|none") + options.define("log_to_stderr", type=bool, default=None, + help=("Send log output to stderr (colorized if possible). " + "By default use stderr if --log_file_prefix is not set and " + "no other logging is configured.")) + options.define("log_file_prefix", type=str, default=None, metavar="PATH", + help=("Path prefix for log files. " + "Note that if you are running multiple tornado processes, " + "log_file_prefix must be different for each of them (e.g. " + "include the port number)")) + options.define("log_file_max_size", type=int, default=100 * 1000 * 1000, + help="max size of log files before rollover") + options.define("log_file_num_backups", type=int, default=10, + help="number of log files to keep") + + options.add_parse_callback(enable_pretty_logging) diff -Nru python-tornado-2.1.0/tornado/netutil.py python-tornado-3.1.1/tornado/netutil.py --- python-tornado-2.1.0/tornado/netutil.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/netutil.py 2013-09-01 18:41:35.000000000 +0000 @@ -16,208 +16,22 @@ """Miscellaneous network utility code.""" +from __future__ import absolute_import, division, print_function, with_statement + import errno -import logging import os +import re import socket +import ssl import stat -from tornado import process +from tornado.concurrent import dummy_executor, run_on_executor from tornado.ioloop import IOLoop -from tornado.iostream import IOStream, SSLIOStream from tornado.platform.auto import set_close_exec +from tornado.util import Configurable -try: - import ssl # Python 2.6+ -except ImportError: - ssl = None - -class TCPServer(object): - r"""A non-blocking, single-threaded TCP server. - - To use `TCPServer`, define a subclass which overrides the `handle_stream` - method. - - `TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. - To make this server serve SSL traffic, send the ssl_options dictionary - argument with the arguments required for the `ssl.wrap_socket` method, - including "certfile" and "keyfile":: - - TCPServer(ssl_options={ - "certfile": os.path.join(data_dir, "mydomain.crt"), - "keyfile": os.path.join(data_dir, "mydomain.key"), - }) - - `TCPServer` initialization follows one of three patterns: - - 1. `listen`: simple single-process:: - - server = TCPServer() - server.listen(8888) - IOLoop.instance().start() - - 2. `bind`/`start`: simple multi-process:: - - server = TCPServer() - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.instance().start() - - When using this interface, an `IOLoop` must *not* be passed - to the `TCPServer` constructor. `start` will always start - the server on the default singleton `IOLoop`. - - 3. `add_sockets`: advanced multi-process:: - - sockets = bind_sockets(8888) - tornado.process.fork_processes(0) - server = TCPServer() - server.add_sockets(sockets) - IOLoop.instance().start() - - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `bind_sockets`. - """ - def __init__(self, io_loop=None, ssl_options=None): - self.io_loop = io_loop - self.ssl_options = ssl_options - self._sockets = {} # fd -> socket object - self._pending_sockets = [] - self._started = False - - def listen(self, port, address=""): - """Starts accepting connections on the given port. - - This method may be called more than once to listen on multiple ports. - `listen` takes effect immediately; it is not necessary to call - `TCPServer.start` afterwards. It is, however, necessary to start - the `IOLoop`. - """ - sockets = bind_sockets(port, address=address) - self.add_sockets(sockets) - - def add_sockets(self, sockets): - """Makes this server start accepting connections on the given sockets. - - The ``sockets`` parameter is a list of socket objects such as - those returned by `bind_sockets`. - `add_sockets` is typically used in combination with that - method and `tornado.process.fork_processes` to provide greater - control over the initialization of a multi-process server. - """ - if self.io_loop is None: - self.io_loop = IOLoop.instance() - - for sock in sockets: - self._sockets[sock.fileno()] = sock - add_accept_handler(sock, self._handle_connection, - io_loop=self.io_loop) - - def add_socket(self, socket): - """Singular version of `add_sockets`. Takes a single socket object.""" - self.add_sockets([socket]) - - def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128): - """Binds this server to the given port on the given address. - - To start the server, call `start`. If you want to run this server - in a single process, you can call `listen` as a shortcut to the - sequence of `bind` and `start` calls. - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either ``socket.AF_INET`` - or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen`. - This method may be called multiple times prior to `start` to listen - on multiple ports or interfaces. - """ - sockets = bind_sockets(port, address=address, family=family, - backlog=backlog) - if self._started: - self.add_sockets(sockets) - else: - self._pending_sockets.extend(sockets) - - def start(self, num_processes=1): - """Starts this server in the IOLoop. - - By default, we run the server in this process and do not fork any - additional child process. - - If num_processes is ``None`` or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If num_processes is given and > 1, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``debug=True`` option to `tornado.web.Application`). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``TCPServer.start(n)``. - """ - assert not self._started - self._started = True - if num_processes != 1: - process.fork_processes(num_processes) - sockets = self._pending_sockets - self._pending_sockets = [] - self.add_sockets(sockets) - - def stop(self): - """Stops listening for new connections. - - Requests currently in progress may still continue after the - server is stopped. - """ - for fd, sock in self._sockets.iteritems(): - self.io_loop.remove_handler(fd) - sock.close() - - def handle_stream(self, stream, address): - """Override to handle a new `IOStream` from an incoming connection.""" - raise NotImplementedError() - - def _handle_connection(self, connection, address): - if self.ssl_options is not None: - assert ssl, "Python 2.6+ and OpenSSL required for SSL" - try: - connection = ssl.wrap_socket(connection, - server_side=True, - do_handshake_on_connect=False, - **self.ssl_options) - except ssl.SSLError, err: - if err.args[0] == ssl.SSL_ERROR_EOF: - return connection.close() - else: - raise - except socket.error, err: - if err.args[0] == errno.ECONNABORTED: - return connection.close() - else: - raise - try: - if self.ssl_options is not None: - stream = SSLIOStream(connection, io_loop=self.io_loop) - else: - stream = IOStream(connection, io_loop=self.io_loop) - self.handle_stream(stream, address) - except Exception: - logging.error("Error in connection callback", exc_info=True) - - -def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128): +def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. Returns a list of socket objects (multiple sockets are returned if @@ -227,29 +41,40 @@ Address may be either an IP address or hostname. If it's a hostname, the server will listen on all IP addresses associated with the name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either socket.AF_INET - or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise + available interfaces. Family may be set to either `socket.AF_INET` + or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise both will be used if available. - The ``backlog`` argument has the same meaning as for - ``socket.listen()``. + The ``backlog`` argument has the same meaning as for + `socket.listen() `. + + ``flags`` is a bitmask of AI_* flags to `~socket.getaddrinfo`, like + ``socket.AI_PASSIVE | socket.AI_NUMERICHOST``. """ sockets = [] if address == "": address = None - flags = socket.AI_PASSIVE - if hasattr(socket, "AI_ADDRCONFIG"): - # AI_ADDRCONFIG ensures that we only try to bind on ipv6 - # if the system is configured for it, but the flag doesn't - # exist on some platforms (specifically WinXP, although - # newer versions of windows have it) - flags |= socket.AI_ADDRCONFIG - for res in socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, - 0, flags): + if not socket.has_ipv6 and family == socket.AF_UNSPEC: + # Python can be compiled with --disable-ipv6, which causes + # operations on AF_INET6 sockets to fail, but does not + # automatically exclude those results from getaddrinfo + # results. + # http://bugs.python.org/issue16208 + family = socket.AF_INET + if flags is None: + flags = socket.AI_PASSIVE + for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, + 0, flags)): af, socktype, proto, canonname, sockaddr = res - sock = socket.socket(af, socktype, proto) + try: + sock = socket.socket(af, socktype, proto) + except socket.error as e: + if e.args[0] == errno.EAFNOSUPPORT: + continue + raise set_close_exec(sock.fileno()) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if os.name != 'nt': + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if af == socket.AF_INET6: # On linux, ipv6 sockets accept ipv4 too by default, # but this makes it impossible to bind to both @@ -268,14 +93,14 @@ return sockets if hasattr(socket, 'AF_UNIX'): - def bind_unix_socket(file, mode=0600, backlog=128): + def bind_unix_socket(file, mode=0o600, backlog=128): """Creates a listening unix socket. If a socket with the given name already exists, it will be deleted. If any other file with that name exists, an exception will be raised. - Returns a socket object (not a list of socket objects like + Returns a socket object (not a list of socket objects like `bind_sockets`) """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -284,7 +109,7 @@ sock.setblocking(0) try: st = os.stat(file) - except OSError, err: + except OSError as err: if err.errno != errno.ENOENT: raise else: @@ -297,24 +122,338 @@ sock.listen(backlog) return sock + def add_accept_handler(sock, callback, io_loop=None): - """Adds an ``IOLoop`` event handler to accept new connections on ``sock``. + """Adds an `.IOLoop` event handler to accept new connections on ``sock``. When a connection is accepted, ``callback(connection, address)`` will be run (``connection`` is a socket object, and ``address`` is the address of the other end of the connection). Note that this signature is different from the ``callback(fd, events)`` signature used for - ``IOLoop`` handlers. + `.IOLoop` handlers. """ if io_loop is None: - io_loop = IOLoop.instance() + io_loop = IOLoop.current() + def accept_handler(fd, events): while True: try: connection, address = sock.accept() - except socket.error, e: + except socket.error as e: + # EWOULDBLOCK and EAGAIN indicate we have accepted every + # connection that is available. if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): return + # ECONNABORTED indicates that there was a connection + # but it was closed while still in the accept queue. + # (observed on FreeBSD). + if e.args[0] == errno.ECONNABORTED: + continue raise callback(connection, address) io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ) + + +def is_valid_ip(ip): + """Returns true if the given string is a well-formed IP address. + + Supports IPv4 and IPv6. + """ + try: + res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, + socket.SOCK_STREAM, + 0, socket.AI_NUMERICHOST) + return bool(res) + except socket.gaierror as e: + if e.args[0] == socket.EAI_NONAME: + return False + raise + return True + + +class Resolver(Configurable): + """Configurable asynchronous DNS resolver interface. + + By default, a blocking implementation is used (which simply calls + `socket.getaddrinfo`). An alternative implementation can be + chosen with the `Resolver.configure <.Configurable.configure>` + class method:: + + Resolver.configure('tornado.netutil.ThreadedResolver') + + The implementations of this interface included with Tornado are + + * `tornado.netutil.BlockingResolver` + * `tornado.netutil.ThreadedResolver` + * `tornado.netutil.OverrideResolver` + * `tornado.platform.twisted.TwistedResolver` + * `tornado.platform.caresresolver.CaresResolver` + """ + @classmethod + def configurable_base(cls): + return Resolver + + @classmethod + def configurable_default(cls): + return BlockingResolver + + def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None): + """Resolves an address. + + The ``host`` argument is a string which may be a hostname or a + literal IP address. + + Returns a `.Future` whose result is a list of (family, + address) pairs, where address is a tuple suitable to pass to + `socket.connect ` (i.e. a ``(host, + port)`` pair for IPv4; additional fields may be present for + IPv6). If a ``callback`` is passed, it will be run with the + result as an argument when it is complete. + """ + raise NotImplementedError() + + def close(self): + """Closes the `Resolver`, freeing any resources used. + + .. versionadded:: 3.1 + + """ + pass + + +class ExecutorResolver(Resolver): + """Resolver implementation using a `concurrent.futures.Executor`. + + Use this instead of `ThreadedResolver` when you require additional + control over the executor being used. + + The executor will be shut down when the resolver is closed unless + ``close_resolver=False``; use this if you want to reuse the same + executor elsewhere. + """ + def initialize(self, io_loop=None, executor=None, close_executor=True): + self.io_loop = io_loop or IOLoop.current() + if executor is not None: + self.executor = executor + self.close_executor = close_executor + else: + self.executor = dummy_executor + self.close_executor = False + + def close(self): + if self.close_executor: + self.executor.shutdown() + self.executor = None + + @run_on_executor + def resolve(self, host, port, family=socket.AF_UNSPEC): + # On Solaris, getaddrinfo fails if the given port is not found + # in /etc/services and no socket type is given, so we must pass + # one here. The socket type used here doesn't seem to actually + # matter (we discard the one we get back in the results), + # so the addresses we return should still be usable with SOCK_DGRAM. + addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) + results = [] + for family, socktype, proto, canonname, address in addrinfo: + results.append((family, address)) + return results + + +class BlockingResolver(ExecutorResolver): + """Default `Resolver` implementation, using `socket.getaddrinfo`. + + The `.IOLoop` will be blocked during the resolution, although the + callback will not be run until the next `.IOLoop` iteration. + """ + def initialize(self, io_loop=None): + super(BlockingResolver, self).initialize(io_loop=io_loop) + + +class ThreadedResolver(ExecutorResolver): + """Multithreaded non-blocking `Resolver` implementation. + + Requires the `concurrent.futures` package to be installed + (available in the standard library since Python 3.2, + installable with ``pip install futures`` in older versions). + + The thread pool size can be configured with:: + + Resolver.configure('tornado.netutil.ThreadedResolver', + num_threads=10) + + .. versionchanged:: 3.1 + All ``ThreadedResolvers`` share a single thread pool, whose + size is set by the first one to be created. + """ + _threadpool = None + _threadpool_pid = None + + def initialize(self, io_loop=None, num_threads=10): + threadpool = ThreadedResolver._create_threadpool(num_threads) + super(ThreadedResolver, self).initialize( + io_loop=io_loop, executor=threadpool, close_executor=False) + + @classmethod + def _create_threadpool(cls, num_threads): + pid = os.getpid() + if cls._threadpool_pid != pid: + # Threads cannot survive after a fork, so if our pid isn't what it + # was when we created the pool then delete it. + cls._threadpool = None + if cls._threadpool is None: + from concurrent.futures import ThreadPoolExecutor + cls._threadpool = ThreadPoolExecutor(num_threads) + cls._threadpool_pid = pid + return cls._threadpool + + +class OverrideResolver(Resolver): + """Wraps a resolver with a mapping of overrides. + + This can be used to make local DNS changes (e.g. for testing) + without modifying system-wide settings. + + The mapping can contain either host strings or host-port pairs. + """ + def initialize(self, resolver, mapping): + self.resolver = resolver + self.mapping = mapping + + def close(self): + self.resolver.close() + + def resolve(self, host, port, *args, **kwargs): + if (host, port) in self.mapping: + host, port = self.mapping[(host, port)] + elif host in self.mapping: + host = self.mapping[host] + return self.resolver.resolve(host, port, *args, **kwargs) + + +# These are the keyword arguments to ssl.wrap_socket that must be translated +# to their SSLContext equivalents (the other arguments are still passed +# to SSLContext.wrap_socket). +_SSL_CONTEXT_KEYWORDS = frozenset(['ssl_version', 'certfile', 'keyfile', + 'cert_reqs', 'ca_certs', 'ciphers']) + + +def ssl_options_to_context(ssl_options): + """Try to convert an ``ssl_options`` dictionary to an + `~ssl.SSLContext` object. + + The ``ssl_options`` dictionary contains keywords to be passed to + `ssl.wrap_socket`. In Python 3.2+, `ssl.SSLContext` objects can + be used instead. This function converts the dict form to its + `~ssl.SSLContext` equivalent, and may be used when a component which + accepts both forms needs to upgrade to the `~ssl.SSLContext` version + to use features like SNI or NPN. + """ + if isinstance(ssl_options, dict): + assert all(k in _SSL_CONTEXT_KEYWORDS for k in ssl_options), ssl_options + if (not hasattr(ssl, 'SSLContext') or + isinstance(ssl_options, ssl.SSLContext)): + return ssl_options + context = ssl.SSLContext( + ssl_options.get('ssl_version', ssl.PROTOCOL_SSLv23)) + if 'certfile' in ssl_options: + context.load_cert_chain(ssl_options['certfile'], ssl_options.get('keyfile', None)) + if 'cert_reqs' in ssl_options: + context.verify_mode = ssl_options['cert_reqs'] + if 'ca_certs' in ssl_options: + context.load_verify_locations(ssl_options['ca_certs']) + if 'ciphers' in ssl_options: + context.set_ciphers(ssl_options['ciphers']) + return context + + +def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): + """Returns an ``ssl.SSLSocket`` wrapping the given socket. + + ``ssl_options`` may be either a dictionary (as accepted by + `ssl_options_to_context`) or an `ssl.SSLContext` object. + Additional keyword arguments are passed to ``wrap_socket`` + (either the `~ssl.SSLContext` method or the `ssl` module function + as appropriate). + """ + context = ssl_options_to_context(ssl_options) + if hasattr(ssl, 'SSLContext') and isinstance(context, ssl.SSLContext): + if server_hostname is not None and getattr(ssl, 'HAS_SNI'): + # Python doesn't have server-side SNI support so we can't + # really unittest this, but it can be manually tested with + # python3.2 -m tornado.httpclient https://sni.velox.ch + return context.wrap_socket(socket, server_hostname=server_hostname, + **kwargs) + else: + return context.wrap_socket(socket, **kwargs) + else: + return ssl.wrap_socket(socket, **dict(context, **kwargs)) + +if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ + ssl_match_hostname = ssl.match_hostname + SSLCertificateError = ssl.CertificateError +else: + # match_hostname was added to the standard library ssl module in python 3.2. + # The following code was backported for older releases and copied from + # https://bitbucket.org/brandon/backports.ssl_match_hostname + class SSLCertificateError(ValueError): + pass + + def _dnsname_to_pat(dn, max_wildcards=1): + pats = [] + for frag in dn.split(r'.'): + if frag.count('*') > max_wildcards: + # Issue #17980: avoid denials of service by refusing more + # than one wildcard per fragment. A survery of established + # policy among SSL implementations showed it to be a + # reasonable choice. + raise SSLCertificateError( + "too many wildcards in certificate DNS name: " + repr(dn)) + if frag == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + else: + # Otherwise, '*' matches any dotless fragment. + frag = re.escape(frag) + pats.append(frag.replace(r'\*', '[^.]*')) + return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + + def ssl_match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules + are mostly followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError("empty or no certificate") + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if not dnsnames: + # The subject is only checked when there is no dNSName entry + # in subjectAltName + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if len(dnsnames) > 1: + raise SSLCertificateError("hostname %r " + "doesn't match either of %s" + % (hostname, ', '.join(map(repr, dnsnames)))) + elif len(dnsnames) == 1: + raise SSLCertificateError("hostname %r " + "doesn't match %r" + % (hostname, dnsnames[0])) + else: + raise SSLCertificateError("no appropriate commonName or " + "subjectAltName fields were found") diff -Nru python-tornado-2.1.0/tornado/options.py python-tornado-3.1.1/tornado/options.py --- python-tornado-2.1.0/tornado/options.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/options.py 2013-08-04 19:34:21.000000000 +0000 @@ -16,7 +16,8 @@ """A command line parsing module that lets modules define their own options. -Each module defines its own options, e.g.:: +Each module defines its own options which are added to the global +option namespace, e.g.:: from tornado.options import define, options @@ -28,173 +29,340 @@ db = database.Connection(options.mysql_host) ... -The main() method of your application does not need to be aware of all of +The ``main()`` method of your application does not need to be aware of all of the options used throughout your program; they are all automatically loaded -when the modules are loaded. Your main() method can parse the command line -or parse a config file with:: +when the modules are loaded. However, all modules that define options +must have been imported before the command line is parsed. + +Your ``main()`` method can parse the command line or parse a config file with +either:: - import tornado.options - tornado.options.parse_config_file("/etc/server.conf") tornado.options.parse_command_line() + # or + tornado.options.parse_config_file("/etc/server.conf") -Command line formats are what you would expect ("--myoption=myvalue"). +Command line formats are what you would expect (``--myoption=myvalue``). Config files are just Python files. Global names become options, e.g.:: myoption = "myvalue" myotheroption = "myothervalue" -We support datetimes, timedeltas, ints, and floats (just pass a 'type' -kwarg to define). We also accept multi-value options. See the documentation -for define() below. +We support `datetimes `, `timedeltas +`, ints, and floats (just pass a ``type`` kwarg to +`define`). We also accept multi-value options. See the documentation for +`define()` below. + +`tornado.options.options` is a singleton instance of `OptionParser`, and +the top-level functions in this module (`define`, `parse_command_line`, etc) +simply call methods on it. You may create additional `OptionParser` +instances to define isolated sets of options, such as for subcommands. """ +from __future__ import absolute_import, division, print_function, with_statement + import datetime -import logging -import logging.handlers +import numbers import re import sys -import time +import os +import textwrap from tornado.escape import _unicode +from tornado.log import define_logging_options +from tornado import stack_context +from tornado.util import basestring_type, exec_in -# For pretty log messages, if available -try: - import curses -except ImportError: - curses = None +class Error(Exception): + """Exception raised by errors in the options module.""" + pass -def define(name, default=None, type=None, help=None, metavar=None, - multiple=False, group=None): - """Defines a new command line option. - If type is given (one of str, float, int, datetime, or timedelta) - or can be inferred from the default, we parse the command line - arguments based on the given type. If multiple is True, we accept - comma-separated values, and the option value is always a list. +class OptionParser(object): + """A collection of options, a dictionary with object-like access. + + Normally accessed via static functions in the `tornado.options` module, + which reference a global instance. + """ + def __init__(self): + # we have to use self.__dict__ because we override setattr. + self.__dict__['_options'] = {} + self.__dict__['_parse_callbacks'] = [] + self.define("help", type=bool, help="show this help information", + callback=self._help_callback) + + def __getattr__(self, name): + if isinstance(self._options.get(name), _Option): + return self._options[name].value() + raise AttributeError("Unrecognized option %r" % name) - For multi-value integers, we also accept the syntax x:y, which - turns into range(x, y) - very useful for long integer ranges. + def __setattr__(self, name, value): + if isinstance(self._options.get(name), _Option): + return self._options[name].set(value) + raise AttributeError("Unrecognized option %r" % name) - help and metavar are used to construct the automatically generated - command line help string. The help message is formatted like:: + def __iter__(self): + return iter(self._options) - --name=METAVAR help string + def __getitem__(self, item): + return self._options[item].value() - group is used to group the defined options in logical groups. By default, - command line options are grouped by the defined file. + def items(self): + """A sequence of (name, value) pairs. - Command line option names must be unique globally. They can be parsed - from the command line with parse_command_line() or parsed from a - config file with parse_config_file. - """ - if name in options: - raise Error("Option %r already defined in %s", name, - options[name].file_name) - frame = sys._getframe(0) - options_file = frame.f_code.co_filename - file_name = frame.f_back.f_code.co_filename - if file_name == options_file: file_name = "" - if type is None: - if not multiple and default is not None: - type = default.__class__ + .. versionadded:: 3.1 + """ + return [(name, opt.value()) for name, opt in self._options.items()] + + def groups(self): + """The set of option-groups created by ``define``. + + .. versionadded:: 3.1 + """ + return set(opt.group_name for opt in self._options.values()) + + def group_dict(self, group): + """The names and values of options in a group. + + Useful for copying options into Application settings:: + + from tornado.options import define, parse_command_line, options + + define('template_path', group='application') + define('static_path', group='application') + + parse_command_line() + + application = Application( + handlers, **options.group_dict('application')) + + .. versionadded:: 3.1 + """ + return dict( + (name, opt.value()) for name, opt in self._options.items() + if not group or group == opt.group_name) + + def as_dict(self): + """The names and values of all options. + + .. versionadded:: 3.1 + """ + return dict( + (name, opt.value()) for name, opt in self._options.items()) + + def define(self, name, default=None, type=None, help=None, metavar=None, + multiple=False, group=None, callback=None): + """Defines a new command line option. + + If ``type`` is given (one of str, float, int, datetime, or timedelta) + or can be inferred from the ``default``, we parse the command line + arguments based on the given type. If ``multiple`` is True, we accept + comma-separated values, and the option value is always a list. + + For multi-value integers, we also accept the syntax ``x:y``, which + turns into ``range(x, y)`` - very useful for long integer ranges. + + ``help`` and ``metavar`` are used to construct the + automatically generated command line help string. The help + message is formatted like:: + + --name=METAVAR help string + + ``group`` is used to group the defined options in logical + groups. By default, command line options are grouped by the + file in which they are defined. + + Command line option names must be unique globally. They can be parsed + from the command line with `parse_command_line` or parsed from a + config file with `parse_config_file`. + + If a ``callback`` is given, it will be run with the new value whenever + the option is changed. This can be used to combine command-line + and file-based options:: + + define("config", type=str, help="path to config file", + callback=lambda path: parse_config_file(path, final=False)) + + With this definition, options in the file specified by ``--config`` will + override options set earlier on the command line, but can be overridden + by later flags. + """ + if name in self._options: + raise Error("Option %r already defined in %s" % + (name, self._options[name].file_name)) + frame = sys._getframe(0) + options_file = frame.f_code.co_filename + file_name = frame.f_back.f_code.co_filename + if file_name == options_file: + file_name = "" + if type is None: + if not multiple and default is not None: + type = default.__class__ + else: + type = str + if group: + group_name = group else: - type = str - if group: - group_name = group - else: - group_name = file_name - options[name] = _Option(name, file_name=file_name, default=default, - type=type, help=help, metavar=metavar, - multiple=multiple, group_name=group_name) + group_name = file_name + self._options[name] = _Option(name, file_name=file_name, + default=default, type=type, help=help, + metavar=metavar, multiple=multiple, + group_name=group_name, + callback=callback) + + def parse_command_line(self, args=None, final=True): + """Parses all options given on the command line (defaults to + `sys.argv`). + + Note that ``args[0]`` is ignored since it is the program name + in `sys.argv`. + + We return a list of all arguments that are not parsed as options. + + If ``final`` is ``False``, parse callbacks will not be run. + This is useful for applications that wish to combine configurations + from multiple sources. + """ + if args is None: + args = sys.argv + remaining = [] + for i in range(1, len(args)): + # All things after the last option are command line arguments + if not args[i].startswith("-"): + remaining = args[i:] + break + if args[i] == "--": + remaining = args[i + 1:] + break + arg = args[i].lstrip("-") + name, equals, value = arg.partition("=") + name = name.replace('-', '_') + if not name in self._options: + self.print_help() + raise Error('Unrecognized command line option: %r' % name) + option = self._options[name] + if not equals: + if option.type == bool: + value = "true" + else: + raise Error('Option %r requires a value' % name) + option.parse(value) + if final: + self.run_parse_callbacks() -def parse_command_line(args=None): - """Parses all options given on the command line. + return remaining - We return all command line arguments that are not options as a list. + def parse_config_file(self, path, final=True): + """Parses and loads the Python config file at the given path. + + If ``final`` is ``False``, parse callbacks will not be run. + This is useful for applications that wish to combine configurations + from multiple sources. + """ + config = {} + with open(path) as f: + exec_in(f.read(), config, config) + for name in config: + if name in self._options: + self._options[name].set(config[name]) + + if final: + self.run_parse_callbacks() + + def print_help(self, file=None): + """Prints all the command line options to stderr (or another file).""" + if file is None: + file = sys.stderr + print("Usage: %s [OPTIONS]" % sys.argv[0], file=file) + print("\nOptions:\n", file=file) + by_group = {} + for option in self._options.values(): + by_group.setdefault(option.group_name, []).append(option) + + for filename, o in sorted(by_group.items()): + if filename: + print("\n%s options:\n" % os.path.normpath(filename), file=file) + o.sort(key=lambda option: option.name) + for option in o: + prefix = option.name + if option.metavar: + prefix += "=" + option.metavar + description = option.help or "" + if option.default is not None and option.default != '': + description += " (default %s)" % option.default + lines = textwrap.wrap(description, 79 - 35) + if len(prefix) > 30 or len(lines) == 0: + lines.insert(0, '') + print(" --%-30s %s" % (prefix, lines[0]), file=file) + for line in lines[1:]: + print("%-34s %s" % (' ', line), file=file) + print(file=file) + + def _help_callback(self, value): + if value: + self.print_help() + sys.exit(0) + + def add_parse_callback(self, callback): + """Adds a parse callback, to be invoked when option parsing is done.""" + self._parse_callbacks.append(stack_context.wrap(callback)) + + def run_parse_callbacks(self): + for callback in self._parse_callbacks: + callback() + + def mockable(self): + """Returns a wrapper around self that is compatible with + `mock.patch `. + + The `mock.patch ` function (included in + the standard library `unittest.mock` package since Python 3.3, + or in the third-party ``mock`` package for older versions of + Python) is incompatible with objects like ``options`` that + override ``__getattr__`` and ``__setattr__``. This function + returns an object that can be used with `mock.patch.object + ` to modify option values:: + + with mock.patch.object(options.mockable(), 'name', value): + assert options.name == value + """ + return _Mockable(self) + + +class _Mockable(object): + """`mock.patch` compatible wrapper for `OptionParser`. + + As of ``mock`` version 1.0.1, when an object uses ``__getattr__`` + hooks instead of ``__dict__``, ``patch.__exit__`` tries to delete + the attribute it set instead of setting a new one (assuming that + the object does not catpure ``__setattr__``, so the patch + created a new attribute in ``__dict__``). + + _Mockable's getattr and setattr pass through to the underlying + OptionParser, and delattr undoes the effect of a previous setattr. """ - if args is None: args = sys.argv - remaining = [] - for i in xrange(1, len(args)): - # All things after the last option are command line arguments - if not args[i].startswith("-"): - remaining = args[i:] - break - if args[i] == "--": - remaining = args[i+1:] - break - arg = args[i].lstrip("-") - name, equals, value = arg.partition("=") - name = name.replace('-', '_') - if not name in options: - print_help() - raise Error('Unrecognized command line option: %r' % name) - option = options[name] - if not equals: - if option.type == bool: - value = "true" - else: - raise Error('Option %r requires a value' % name) - option.parse(value) - if options.help: - print_help() - sys.exit(0) - - # Set up log level and pretty console logging by default - if options.logging != 'none': - logging.getLogger().setLevel(getattr(logging, options.logging.upper())) - enable_pretty_logging() - - return remaining - - -def parse_config_file(path): - """Parses and loads the Python config file at the given path.""" - config = {} - execfile(path, config, config) - for name in config: - if name in options: - options[name].set(config[name]) - - -def print_help(file=sys.stdout): - """Prints all the command line options to stdout.""" - print >> file, "Usage: %s [OPTIONS]" % sys.argv[0] - print >> file, "" - print >> file, "Options:" - by_group = {} - for option in options.itervalues(): - by_group.setdefault(option.group_name, []).append(option) - - for filename, o in sorted(by_group.items()): - if filename: print >> file, filename - o.sort(key=lambda option: option.name) - for option in o: - prefix = option.name - if option.metavar: - prefix += "=" + option.metavar - print >> file, " --%-30s %s" % (prefix, option.help or "") - print >> file - - -class _Options(dict): - """Our global program options, an dictionary with object-like access.""" - @classmethod - def instance(cls): - if not hasattr(cls, "_instance"): - cls._instance = cls() - return cls._instance + def __init__(self, options): + # Modify __dict__ directly to bypass __setattr__ + self.__dict__['_options'] = options + self.__dict__['_originals'] = {} def __getattr__(self, name): - if isinstance(self.get(name), _Option): - return self[name].value() - raise AttributeError("Unrecognized option %r" % name) + return getattr(self._options, name) + + def __setattr__(self, name, value): + assert name not in self._originals, "don't reuse mockable objects" + self._originals[name] = getattr(self._options, name) + setattr(self._options, name, value) + + def __delattr__(self, name): + setattr(self._options, name, self._originals.pop(name)) class _Option(object): - def __init__(self, name, default=None, type=str, help=None, metavar=None, - multiple=False, file_name=None, group_name=None): + def __init__(self, name, default=None, type=basestring_type, help=None, + metavar=None, multiple=False, file_name=None, group_name=None, + callback=None): if default is None and multiple: default = [] self.name = name @@ -204,6 +372,7 @@ self.multiple = multiple self.file_name = file_name self.group_name = group_name + self.callback = callback self.default = default self._value = None @@ -215,22 +384,23 @@ datetime.datetime: self._parse_datetime, datetime.timedelta: self._parse_timedelta, bool: self._parse_bool, - str: self._parse_string, + basestring_type: self._parse_string, }.get(self.type, self.type) if self.multiple: - if self._value is None: - self._value = [] + self._value = [] for part in value.split(","): - if self.type in (int, long): + if issubclass(self.type, numbers.Integral): # allow ranges of the form X:Y (inclusive at both ends) lo, _, hi = part.partition(":") lo = _parse(lo) hi = _parse(hi) if hi else lo - self._value.extend(range(lo, hi+1)) + self._value.extend(range(lo, hi + 1)) else: self._value.append(_parse(part)) else: self._value = _parse(value) + if self.callback is not None: + self.callback(self._value) return self.value() def set(self, value): @@ -239,14 +409,16 @@ raise Error("Option %r is required to be a list of %s" % (self.name, self.type.__name__)) for item in value: - if item != None and not isinstance(item, self.type): + if item is not None and not isinstance(item, self.type): raise Error("Option %r is required to be a list of %s" % (self.name, self.type.__name__)) else: - if value != None and not isinstance(value, self.type): - raise Error("Option %r is required to be a %s" % - (self.name, self.type.__name__)) + if value is not None and not isinstance(value, self.type): + raise Error("Option %r is required to be a %s (%s given)" % + (self.name, self.type.__name__, type(value))) self._value = value + if self.callback is not None: + self.callback(self._value) # Supported date/time formats in our options _DATETIME_FORMATS = [ @@ -313,105 +485,55 @@ return _unicode(value) -class Error(Exception): - """Exception raised by errors in the options module.""" - pass +options = OptionParser() +"""Global options object. + +All defined options are available as attributes on this object. +""" -def enable_pretty_logging(): - """Turns on formatted logging output as configured. - - This is called automatically by `parse_command_line`. +def define(name, default=None, type=None, help=None, metavar=None, + multiple=False, group=None, callback=None): + """Defines an option in the global namespace. + + See `OptionParser.define`. """ - root_logger = logging.getLogger() - if options.log_file_prefix: - channel = logging.handlers.RotatingFileHandler( - filename=options.log_file_prefix, - maxBytes=options.log_file_max_size, - backupCount=options.log_file_num_backups) - channel.setFormatter(_LogFormatter(color=False)) - root_logger.addHandler(channel) - - if (options.log_to_stderr or - (options.log_to_stderr is None and not root_logger.handlers)): - # Set up color if we are in a tty and curses is installed - color = False - if curses and sys.stderr.isatty(): - try: - curses.setupterm() - if curses.tigetnum("colors") > 0: - color = True - except Exception: - pass - channel = logging.StreamHandler() - channel.setFormatter(_LogFormatter(color=color)) - root_logger.addHandler(channel) + return options.define(name, default=default, type=type, help=help, + metavar=metavar, multiple=multiple, group=group, + callback=callback) +def parse_command_line(args=None, final=True): + """Parses global options from the command line. -class _LogFormatter(logging.Formatter): - def __init__(self, color, *args, **kwargs): - logging.Formatter.__init__(self, *args, **kwargs) - self._color = color - if color: - # The curses module has some str/bytes confusion in python3. - # Most methods return bytes, but only accept strings. - # The explict calls to unicode() below are harmless in python2, - # but will do the right conversion in python3. - fg_color = unicode(curses.tigetstr("setaf") or - curses.tigetstr("setf") or "", "ascii") - self._colors = { - logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue - "ascii"), - logging.INFO: unicode(curses.tparm(fg_color, 2), # Green - "ascii"), - logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow - "ascii"), - logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red - "ascii"), - } - self._normal = unicode(curses.tigetstr("sgr0"), "ascii") + See `OptionParser.parse_command_line`. + """ + return options.parse_command_line(args, final=final) + + +def parse_config_file(path, final=True): + """Parses global options from a config file. + + See `OptionParser.parse_config_file`. + """ + return options.parse_config_file(path, final=final) - def format(self, record): - try: - record.message = record.getMessage() - except Exception, e: - record.message = "Bad message (%r): %r" % (e, record.__dict__) - record.asctime = time.strftime( - "%y%m%d %H:%M:%S", self.converter(record.created)) - prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ - record.__dict__ - if self._color: - prefix = (self._colors.get(record.levelno, self._normal) + - prefix + self._normal) - formatted = prefix + " " + record.message - if record.exc_info: - if not record.exc_text: - record.exc_text = self.formatException(record.exc_info) - if record.exc_text: - formatted = formatted.rstrip() + "\n" + record.exc_text - return formatted.replace("\n", "\n ") +def print_help(file=None): + """Prints all the command line options to stderr (or another file). -options = _Options.instance() + See `OptionParser.print_help`. + """ + return options.print_help(file) + + +def add_parse_callback(callback): + """Adds a parse callback, to be invoked when option parsing is done. + + See `OptionParser.add_parse_callback` + """ + options.add_parse_callback(callback) # Default options -define("help", type=bool, help="show this help information") -define("logging", default="info", - help=("Set the Python log level. If 'none', tornado won't touch the " - "logging configuration."), - metavar="info|warning|error|none") -define("log_to_stderr", type=bool, default=None, - help=("Send log output to stderr (colorized if possible). " - "By default use stderr if --log_file_prefix is not set and " - "no other logging is configured.")) -define("log_file_prefix", type=str, default=None, metavar="PATH", - help=("Path prefix for log files. " - "Note that if you are running multiple tornado processes, " - "log_file_prefix must be different for each of them (e.g. " - "include the port number)")) -define("log_file_max_size", type=int, default=100 * 1000 * 1000, - help="max size of log files before rollover") -define("log_file_num_backups", type=int, default=10, - help="number of log files to keep") +define_logging_options(options) diff -Nru python-tornado-2.1.0/tornado/platform/auto.py python-tornado-3.1.1/tornado/platform/auto.py --- python-tornado-2.1.0/tornado/platform/auto.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/auto.py 2013-08-04 19:34:21.000000000 +0000 @@ -23,9 +23,23 @@ from tornado.platform.auto import set_close_exec """ +from __future__ import absolute_import, division, print_function, with_statement + import os if os.name == 'nt': - from tornado.platform.windows import set_close_exec, Waker + from tornado.platform.common import Waker + from tornado.platform.windows import set_close_exec else: from tornado.platform.posix import set_close_exec, Waker + +try: + # monotime monkey-patches the time module to have a monotonic function + # in versions of python before 3.3. + import monotime +except ImportError: + pass +try: + from time import monotonic as monotonic_time +except ImportError: + monotonic_time = None diff -Nru python-tornado-2.1.0/tornado/platform/caresresolver.py python-tornado-3.1.1/tornado/platform/caresresolver.py --- python-tornado-2.1.0/tornado/platform/caresresolver.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/caresresolver.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,75 @@ +import pycares +import socket + +from tornado import gen +from tornado.ioloop import IOLoop +from tornado.netutil import Resolver, is_valid_ip + + +class CaresResolver(Resolver): + """Name resolver based on the c-ares library. + + This is a non-blocking and non-threaded resolver. It may not produce + the same results as the system resolver, but can be used for non-blocking + resolution when threads cannot be used. + + c-ares fails to resolve some names when ``family`` is ``AF_UNSPEC``, + so it is only recommended for use in ``AF_INET`` (i.e. IPv4). This is + the default for ``tornado.simple_httpclient``, but other libraries + may default to ``AF_UNSPEC``. + """ + def initialize(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb) + self.fds = {} + + def _sock_state_cb(self, fd, readable, writable): + state = ((IOLoop.READ if readable else 0) | + (IOLoop.WRITE if writable else 0)) + if not state: + self.io_loop.remove_handler(fd) + del self.fds[fd] + elif fd in self.fds: + self.io_loop.update_handler(fd, state) + self.fds[fd] = state + else: + self.io_loop.add_handler(fd, self._handle_events, state) + self.fds[fd] = state + + def _handle_events(self, fd, events): + read_fd = pycares.ARES_SOCKET_BAD + write_fd = pycares.ARES_SOCKET_BAD + if events & IOLoop.READ: + read_fd = fd + if events & IOLoop.WRITE: + write_fd = fd + self.channel.process_fd(read_fd, write_fd) + + @gen.coroutine + def resolve(self, host, port, family=0): + if is_valid_ip(host): + addresses = [host] + else: + # gethostbyname doesn't take callback as a kwarg + self.channel.gethostbyname(host, family, (yield gen.Callback(1))) + callback_args = yield gen.Wait(1) + assert isinstance(callback_args, gen.Arguments) + assert not callback_args.kwargs + result, error = callback_args.args + if error: + raise Exception('C-Ares returned error %s: %s while resolving %s' % + (error, pycares.errno.strerror(error), host)) + addresses = result.addresses + addrinfo = [] + for address in addresses: + if '.' in address: + address_family = socket.AF_INET + elif ':' in address: + address_family = socket.AF_INET6 + else: + address_family = socket.AF_UNSPEC + if family != socket.AF_UNSPEC and family != address_family: + raise Exception('Requested socket family %d but got %d' % + (family, address_family)) + addrinfo.append((address_family, (address, port))) + raise gen.Return(addrinfo) diff -Nru python-tornado-2.1.0/tornado/platform/common.py python-tornado-3.1.1/tornado/platform/common.py --- python-tornado-2.1.0/tornado/platform/common.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/common.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,91 @@ +"""Lowest-common-denominator implementations of platform functionality.""" +from __future__ import absolute_import, division, print_function, with_statement + +import errno +import socket + +from tornado.platform import interface + + +class Waker(interface.Waker): + """Create an OS independent asynchronous pipe. + + For use on platforms that don't have os.pipe() (or where pipes cannot + be passed to select()), but do have sockets. This includes Windows + and Jython. + """ + def __init__(self): + # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py + + self.writer = socket.socket() + # Disable buffering -- pulling the trigger sends 1 byte, + # and we want that sent immediately, to wake up ASAP. + self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + count = 0 + while 1: + count += 1 + # Bind to a local port; for efficiency, let the OS pick + # a free port for us. + # Unfortunately, stress tests showed that we may not + # be able to connect to that port ("Address already in + # use") despite that the OS picked it. This appears + # to be a race bug in the Windows socket implementation. + # So we loop until a connect() succeeds (almost always + # on the first try). See the long thread at + # http://mail.zope.org/pipermail/zope/2005-July/160433.html + # for hideous details. + a = socket.socket() + a.bind(("127.0.0.1", 0)) + a.listen(1) + connect_address = a.getsockname() # assigned (host, port) pair + try: + self.writer.connect(connect_address) + break # success + except socket.error as detail: + if (not hasattr(errno, 'WSAEADDRINUSE') or + detail[0] != errno.WSAEADDRINUSE): + # "Address already in use" is the only error + # I've seen on two WinXP Pro SP2 boxes, under + # Pythons 2.3.5 and 2.4.1. + raise + # (10048, 'Address already in use') + # assert count <= 2 # never triggered in Tim's tests + if count >= 10: # I've never seen it go above 2 + a.close() + self.writer.close() + raise socket.error("Cannot bind trigger!") + # Close `a` and try again. Note: I originally put a short + # sleep() here, but it didn't appear to help or hurt. + a.close() + + self.reader, addr = a.accept() + self.reader.setblocking(0) + self.writer.setblocking(0) + a.close() + self.reader_fd = self.reader.fileno() + + def fileno(self): + return self.reader.fileno() + + def write_fileno(self): + return self.writer.fileno() + + def wake(self): + try: + self.writer.send(b"x") + except (IOError, socket.error): + pass + + def consume(self): + try: + while True: + result = self.reader.recv(1024) + if not result: + break + except (IOError, socket.error): + pass + + def close(self): + self.reader.close() + self.writer.close() diff -Nru python-tornado-2.1.0/tornado/platform/epoll.py python-tornado-3.1.1/tornado/platform/epoll.py --- python-tornado-2.1.0/tornado/platform/epoll.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/epoll.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""EPoll-based IOLoop implementation for Linux systems.""" +from __future__ import absolute_import, division, print_function, with_statement + +import select + +from tornado.ioloop import PollIOLoop + + +class EPollIOLoop(PollIOLoop): + def initialize(self, **kwargs): + super(EPollIOLoop, self).initialize(impl=select.epoll(), **kwargs) diff -Nru python-tornado-2.1.0/tornado/platform/interface.py python-tornado-3.1.1/tornado/platform/interface.py --- python-tornado-2.1.0/tornado/platform/interface.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/interface.py 2013-08-04 19:34:21.000000000 +0000 @@ -21,10 +21,14 @@ implementation from `tornado.platform.auto`. """ +from __future__ import absolute_import, division, print_function, with_statement + + def set_close_exec(fd): """Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor.""" raise NotImplementedError() + class Waker(object): """A socket-like object that can wake another thread from ``select()``. @@ -35,13 +39,17 @@ the ``IOLoop`` is closed, it closes its waker too. """ def fileno(self): - """Returns a file descriptor for this waker. - + """Returns the read file descriptor for this waker. + Must be suitable for use with ``select()`` or equivalent on the local platform. """ raise NotImplementedError() + def write_fileno(self): + """Returns the write file descriptor for this waker.""" + raise NotImplementedError() + def wake(self): """Triggers activity on the waker's file descriptor.""" raise NotImplementedError() @@ -53,5 +61,3 @@ def close(self): """Closes the waker's file descriptor(s).""" raise NotImplementedError() - - diff -Nru python-tornado-2.1.0/tornado/platform/kqueue.py python-tornado-3.1.1/tornado/platform/kqueue.py --- python-tornado-2.1.0/tornado/platform/kqueue.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/kqueue.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""KQueue-based IOLoop implementation for BSD/Mac systems.""" +from __future__ import absolute_import, division, print_function, with_statement + +import select + +from tornado.ioloop import IOLoop, PollIOLoop + +assert hasattr(select, 'kqueue'), 'kqueue not supported' + + +class _KQueue(object): + """A kqueue-based event loop for BSD/Mac systems.""" + def __init__(self): + self._kqueue = select.kqueue() + self._active = {} + + def fileno(self): + return self._kqueue.fileno() + + def close(self): + self._kqueue.close() + + def register(self, fd, events): + if fd in self._active: + raise IOError("fd %d already registered" % fd) + self._control(fd, events, select.KQ_EV_ADD) + self._active[fd] = events + + def modify(self, fd, events): + self.unregister(fd) + self.register(fd, events) + + def unregister(self, fd): + events = self._active.pop(fd) + self._control(fd, events, select.KQ_EV_DELETE) + + def _control(self, fd, events, flags): + kevents = [] + if events & IOLoop.WRITE: + kevents.append(select.kevent( + fd, filter=select.KQ_FILTER_WRITE, flags=flags)) + if events & IOLoop.READ or not kevents: + # Always read when there is not a write + kevents.append(select.kevent( + fd, filter=select.KQ_FILTER_READ, flags=flags)) + # Even though control() takes a list, it seems to return EINVAL + # on Mac OS X (10.6) when there is more than one event in the list. + for kevent in kevents: + self._kqueue.control([kevent], 0) + + def poll(self, timeout): + kevents = self._kqueue.control(None, 1000, timeout) + events = {} + for kevent in kevents: + fd = kevent.ident + if kevent.filter == select.KQ_FILTER_READ: + events[fd] = events.get(fd, 0) | IOLoop.READ + if kevent.filter == select.KQ_FILTER_WRITE: + if kevent.flags & select.KQ_EV_EOF: + # If an asynchronous connection is refused, kqueue + # returns a write event with the EOF flag set. + # Turn this into an error for consistency with the + # other IOLoop implementations. + # Note that for read events, EOF may be returned before + # all data has been consumed from the socket buffer, + # so we only check for EOF on write events. + events[fd] = IOLoop.ERROR + else: + events[fd] = events.get(fd, 0) | IOLoop.WRITE + if kevent.flags & select.KQ_EV_ERROR: + events[fd] = events.get(fd, 0) | IOLoop.ERROR + return events.items() + + +class KQueueIOLoop(PollIOLoop): + def initialize(self, **kwargs): + super(KQueueIOLoop, self).initialize(impl=_KQueue(), **kwargs) diff -Nru python-tornado-2.1.0/tornado/platform/posix.py python-tornado-3.1.1/tornado/platform/posix.py --- python-tornado-2.1.0/tornado/platform/posix.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/posix.py 2013-08-04 19:34:21.000000000 +0000 @@ -16,20 +16,24 @@ """Posix implementations of platform-specific functionality.""" +from __future__ import absolute_import, division, print_function, with_statement + import fcntl import os from tornado.platform import interface -from tornado.util import b + def set_close_exec(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + def _set_nonblocking(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - + + class Waker(interface.Waker): def __init__(self): r, w = os.pipe() @@ -43,9 +47,12 @@ def fileno(self): return self.reader.fileno() + def write_fileno(self): + return self.writer.fileno() + def wake(self): try: - self.writer.write(b("x")) + self.writer.write(b"x") except IOError: pass @@ -53,7 +60,8 @@ try: while True: result = self.reader.read() - if not result: break; + if not result: + break except IOError: pass diff -Nru python-tornado-2.1.0/tornado/platform/select.py python-tornado-3.1.1/tornado/platform/select.py --- python-tornado-2.1.0/tornado/platform/select.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/select.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Select-based IOLoop implementation. + +Used as a fallback for systems that don't support epoll or kqueue. +""" +from __future__ import absolute_import, division, print_function, with_statement + +import select + +from tornado.ioloop import IOLoop, PollIOLoop + + +class _Select(object): + """A simple, select()-based IOLoop implementation for non-Linux systems""" + def __init__(self): + self.read_fds = set() + self.write_fds = set() + self.error_fds = set() + self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) + + def close(self): + pass + + def register(self, fd, events): + if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds: + raise IOError("fd %d already registered" % fd) + if events & IOLoop.READ: + self.read_fds.add(fd) + if events & IOLoop.WRITE: + self.write_fds.add(fd) + if events & IOLoop.ERROR: + self.error_fds.add(fd) + # Closed connections are reported as errors by epoll and kqueue, + # but as zero-byte reads by select, so when errors are requested + # we need to listen for both read and error. + self.read_fds.add(fd) + + def modify(self, fd, events): + self.unregister(fd) + self.register(fd, events) + + def unregister(self, fd): + self.read_fds.discard(fd) + self.write_fds.discard(fd) + self.error_fds.discard(fd) + + def poll(self, timeout): + readable, writeable, errors = select.select( + self.read_fds, self.write_fds, self.error_fds, timeout) + events = {} + for fd in readable: + events[fd] = events.get(fd, 0) | IOLoop.READ + for fd in writeable: + events[fd] = events.get(fd, 0) | IOLoop.WRITE + for fd in errors: + events[fd] = events.get(fd, 0) | IOLoop.ERROR + return events.items() + + +class SelectIOLoop(PollIOLoop): + def initialize(self, **kwargs): + super(SelectIOLoop, self).initialize(impl=_Select(), **kwargs) diff -Nru python-tornado-2.1.0/tornado/platform/twisted.py python-tornado-3.1.1/tornado/platform/twisted.py --- python-tornado-2.1.0/tornado/platform/twisted.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/twisted.py 2013-08-04 19:34:21.000000000 +0000 @@ -13,39 +13,88 @@ # License for the specific language governing permissions and limitations # under the License. -""" -A twisted-style reactor for the Tornado IOLoop. +# Note: This module's docs are not currently extracted automatically, +# so changes must be made manually to twisted.rst +# TODO: refactor doc build process to use an appropriate virtualenv +"""Bridges between the Twisted reactor and Tornado IOLoop. + +This module lets you run applications and libraries written for +Twisted in a Tornado application. It can be used in two modes, +depending on which library's underlying event loop you want to use. + +This module has been tested with Twisted versions 11.0.0 and newer. + +Twisted on Tornado +------------------ + +`TornadoReactor` implements the Twisted reactor interface on top of +the Tornado IOLoop. To use it, simply call `install` at the beginning +of the application:: + + import tornado.platform.twisted + tornado.platform.twisted.install() + from twisted.internet import reactor + +When the app is ready to start, call `IOLoop.instance().start()` +instead of `reactor.run()`. -To use it, add the following to your twisted application: +It is also possible to create a non-global reactor by calling +`tornado.platform.twisted.TornadoReactor(io_loop)`. However, if +the `IOLoop` and reactor are to be short-lived (such as those used in +unit tests), additional cleanup may be required. Specifically, it is +recommended to call:: -import tornado.platform.twisted -tornado.platform.twisted.install() -from twisted.internet import reactor + reactor.fireSystemEvent('shutdown') + reactor.disconnectAll() + +before closing the `IOLoop`. + +Tornado on Twisted +------------------ + +`TwistedIOLoop` implements the Tornado IOLoop interface on top of the Twisted +reactor. Recommended usage:: + + from tornado.platform.twisted import TwistedIOLoop + from twisted.internet import reactor + TwistedIOLoop().install() + # Set up your tornado application as usual using `IOLoop.instance` + reactor.run() + +`TwistedIOLoop` always uses the global Twisted reactor. """ -from __future__ import with_statement, absolute_import +from __future__ import absolute_import, division, print_function, with_statement +import datetime import functools -import logging -import time +import socket +import twisted.internet.abstract from twisted.internet.posixbase import PosixReactorBase from twisted.internet.interfaces import \ - IReactorFDSet, IDelayedCall, IReactorTime + IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor +from twisted.python import failure, log +from twisted.internet import error +import twisted.names.cache +import twisted.names.client +import twisted.names.hosts +import twisted.names.resolve -from zope.interface import implements +from zope.interface import implementer -import tornado +from tornado.escape import utf8 +from tornado import gen import tornado.ioloop -from tornado.stack_context import NullContext +from tornado.log import app_log +from tornado.netutil import Resolver +from tornado.stack_context import NullContext, wrap from tornado.ioloop import IOLoop -class TornadoDelayedCall(object): - """ - DelayedCall object for Tornado. - """ - implements(IDelayedCall) +@implementer(IDelayedCall) +class TornadoDelayedCall(object): + """DelayedCall object for Tornado.""" def __init__(self, reactor, seconds, f, *args, **kw): self._reactor = reactor self._func = functools.partial(f, *args, **kw) @@ -60,7 +109,7 @@ try: self._func() except: - logging.error("_called caught exception", exc_info=True) + app_log.error("_called caught exception", exc_info=True) def getTime(self): return self._time @@ -85,27 +134,40 @@ def active(self): return self._active + +@implementer(IReactorTime, IReactorFDSet) class TornadoReactor(PosixReactorBase): - """ - Twisted style reactor for Tornado. - """ - implements(IReactorTime, IReactorFDSet) + """Twisted reactor built on the Tornado IOLoop. + Since it is intented to be used in applications where the top-level + event loop is ``io_loop.start()`` rather than ``reactor.run()``, + it is implemented a little differently than other Twisted reactors. + We override `mainLoop` instead of `doIteration` and must implement + timed call functionality on top of `IOLoop.add_timeout` rather than + using the implementation in `PosixReactorBase`. + """ def __init__(self, io_loop=None): if not io_loop: - io_loop = tornado.ioloop.IOLoop.instance() + io_loop = tornado.ioloop.IOLoop.current() self._io_loop = io_loop - self._readers = {} - self._writers = {} - self._fds = {} # a map of fd to a (reader, writer) tuple + self._readers = {} # map of reader objects to fd + self._writers = {} # map of writer objects to fd + self._fds = {} # a map of fd to a (reader, writer) tuple self._delayedCalls = {} - self._running = False - self._closed = False PosixReactorBase.__init__(self) + self.addSystemEventTrigger('during', 'shutdown', self.crash) + + # IOLoop.start() bypasses some of the reactor initialization. + # Fire off the necessary events if they weren't already triggered + # by reactor.run(). + def start_if_necessary(): + if not self._started: + self.fireSystemEvent('startup') + self._io_loop.add_callback(start_if_necessary) # IReactorTime def seconds(self): - return time.time() + return self._io_loop.time() def callLater(self, seconds, f, *args, **kw): dc = TornadoDelayedCall(self, seconds, f, *args, **kw) @@ -121,12 +183,14 @@ # IReactorThreads def callFromThread(self, f, *args, **kw): - """ - See L{twisted.internet.interfaces.IReactorThreads.callFromThread}. - """ + """See `twisted.internet.interfaces.IReactorThreads.callFromThread`""" assert callable(f), "%s is not callable" % f - p = functools.partial(f, *args, **kw) - self._io_loop.add_callback(p) + with NullContext(): + # This NullContext is mainly for an edge case when running + # TwistedIOLoop on top of a TornadoReactor. + # TwistedIOLoop.add_callback uses reactor.callFromThread and + # should not pick up additional StackContexts along the way. + self._io_loop.add_callback(f, *args, **kw) # We don't need the waker code from the super class, Tornado uses # its own waker. @@ -138,21 +202,39 @@ # IReactorFDSet def _invoke_callback(self, fd, events): + if fd not in self._fds: + return (reader, writer) = self._fds[fd] - if events | IOLoop.READ and reader: - reader.doRead() - if events | IOLoop.WRITE and writer: - writer.doWrite() + if reader: + err = None + if reader.fileno() == -1: + err = error.ConnectionLost() + elif events & IOLoop.READ: + err = log.callWithLogger(reader, reader.doRead) + if err is None and events & IOLoop.ERROR: + err = error.ConnectionLost() + if err is not None: + self.removeReader(reader) + reader.readConnectionLost(failure.Failure(err)) + if writer: + err = None + if writer.fileno() == -1: + err = error.ConnectionLost() + elif events & IOLoop.WRITE: + err = log.callWithLogger(writer, writer.doWrite) + if err is None and events & IOLoop.ERROR: + err = error.ConnectionLost() + if err is not None: + self.removeWriter(writer) + writer.writeConnectionLost(failure.Failure(err)) def addReader(self, reader): - """ - Add a FileDescriptor for notification of data available to read. - """ + """Add a FileDescriptor for notification of data available to read.""" if reader in self._readers: # Don't add the reader if it's already there return - self._readers[reader] = True fd = reader.fileno() + self._readers[reader] = fd if fd in self._fds: (_, writer) = self._fds[fd] self._fds[fd] = (reader, writer) @@ -164,16 +246,14 @@ with NullContext(): self._fds[fd] = (reader, None) self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.READ) + IOLoop.READ) def addWriter(self, writer): - """ - Add a FileDescriptor for notification of data available to write. - """ + """Add a FileDescriptor for notification of data available to write.""" if writer in self._writers: return - self._writers[writer] = True fd = writer.fileno() + self._writers[writer] = fd if fd in self._fds: (reader, _) = self._fds[fd] self._fds[fd] = (reader, writer) @@ -185,16 +265,12 @@ with NullContext(): self._fds[fd] = (None, writer) self._io_loop.add_handler(fd, self._invoke_callback, - IOLoop.WRITE) + IOLoop.WRITE) def removeReader(self, reader): - """ - Remove a Selectable for notification of data available to read. - """ - fd = reader.fileno() + """Remove a Selectable for notification of data available to read.""" if reader in self._readers: - del self._readers[reader] - if self._closed: return + fd = self._readers.pop(reader) (_, writer) = self._fds[fd] if writer: # We have a writer so we need to update the IOLoop for @@ -209,13 +285,9 @@ self._io_loop.remove_handler(fd) def removeWriter(self, writer): - """ - Remove a Selectable for notification of data available to write. - """ - fd = writer.fileno() + """Remove a Selectable for notification of data available to write.""" if writer in self._writers: - del self._writers[writer] - if self._closed: return + fd = self._writers.pop(writer) (reader, _) = self._fds[fd] if reader: # We have a reader so we need to update the IOLoop for @@ -238,47 +310,30 @@ def getWriters(self): return self._writers.keys() + # The following functions are mainly used in twisted-style test cases; + # it is expected that most users of the TornadoReactor will call + # IOLoop.start() instead of Reactor.run(). def stop(self): - """ - Implement L{IReactorCore.stop}. - """ - self._running = False PosixReactorBase.stop(self) - self.runUntilCurrent() - try: - self._io_loop.stop() - self._io_loop.close() - except: - # Ignore any exceptions thrown by IOLoop - pass - self._closed = True + fire_shutdown = functools.partial(self.fireSystemEvent, "shutdown") + self._io_loop.add_callback(fire_shutdown) def crash(self): - if not self._running: - return - self._running = False PosixReactorBase.crash(self) - self.runUntilCurrent() - try: - self._io_loop.stop() - self._io_loop.close() - except: - # Ignore any exceptions thrown by IOLoop - pass - self._closed = True + self._io_loop.stop() def doIteration(self, delay): raise NotImplementedError("doIteration") def mainLoop(self): - self._running = True self._io_loop.start() + class _TestReactor(TornadoReactor): """Subclass of TornadoReactor for use in unittests. This can't go in the test.py file because of import-order dependencies - with the twisted reactor test builder. + with the Twisted reactor test builder. """ def __init__(self): # always use a new ioloop @@ -291,15 +346,198 @@ return super(_TestReactor, self).listenTCP( port, factory, backlog=backlog, interface=interface) + def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): + if not interface: + interface = '127.0.0.1' + return super(_TestReactor, self).listenUDP( + port, protocol, interface=interface, maxPacketSize=maxPacketSize) def install(io_loop=None): - """ - Install the Tornado reactor. - """ + """Install this package as the default Twisted reactor.""" if not io_loop: - io_loop = tornado.ioloop.IOLoop.instance() + io_loop = tornado.ioloop.IOLoop.current() reactor = TornadoReactor(io_loop) from twisted.internet.main import installReactor installReactor(reactor) return reactor + + +@implementer(IReadDescriptor, IWriteDescriptor) +class _FD(object): + def __init__(self, fd, handler): + self.fd = fd + self.handler = handler + self.reading = False + self.writing = False + self.lost = False + + def fileno(self): + return self.fd + + def doRead(self): + if not self.lost: + self.handler(self.fd, tornado.ioloop.IOLoop.READ) + + def doWrite(self): + if not self.lost: + self.handler(self.fd, tornado.ioloop.IOLoop.WRITE) + + def connectionLost(self, reason): + if not self.lost: + self.handler(self.fd, tornado.ioloop.IOLoop.ERROR) + self.lost = True + + def logPrefix(self): + return '' + + +class TwistedIOLoop(tornado.ioloop.IOLoop): + """IOLoop implementation that runs on Twisted. + + Uses the global Twisted reactor by default. To create multiple + `TwistedIOLoops` in the same process, you must pass a unique reactor + when constructing each one. + + Not compatible with `tornado.process.Subprocess.set_exit_callback` + because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict + with each other. + """ + def initialize(self, reactor=None): + if reactor is None: + import twisted.internet.reactor + reactor = twisted.internet.reactor + self.reactor = reactor + self.fds = {} + self.reactor.callWhenRunning(self.make_current) + + def close(self, all_fds=False): + self.reactor.removeAll() + for c in self.reactor.getDelayedCalls(): + c.cancel() + + def add_handler(self, fd, handler, events): + if fd in self.fds: + raise ValueError('fd %d added twice' % fd) + self.fds[fd] = _FD(fd, wrap(handler)) + if events & tornado.ioloop.IOLoop.READ: + self.fds[fd].reading = True + self.reactor.addReader(self.fds[fd]) + if events & tornado.ioloop.IOLoop.WRITE: + self.fds[fd].writing = True + self.reactor.addWriter(self.fds[fd]) + + def update_handler(self, fd, events): + if events & tornado.ioloop.IOLoop.READ: + if not self.fds[fd].reading: + self.fds[fd].reading = True + self.reactor.addReader(self.fds[fd]) + else: + if self.fds[fd].reading: + self.fds[fd].reading = False + self.reactor.removeReader(self.fds[fd]) + if events & tornado.ioloop.IOLoop.WRITE: + if not self.fds[fd].writing: + self.fds[fd].writing = True + self.reactor.addWriter(self.fds[fd]) + else: + if self.fds[fd].writing: + self.fds[fd].writing = False + self.reactor.removeWriter(self.fds[fd]) + + def remove_handler(self, fd): + if fd not in self.fds: + return + self.fds[fd].lost = True + if self.fds[fd].reading: + self.reactor.removeReader(self.fds[fd]) + if self.fds[fd].writing: + self.reactor.removeWriter(self.fds[fd]) + del self.fds[fd] + + def start(self): + self.reactor.run() + + def stop(self): + self.reactor.crash() + + def _run_callback(self, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception: + self.handle_callback_exception(callback) + + def add_timeout(self, deadline, callback): + if isinstance(deadline, (int, long, float)): + delay = max(deadline - self.time(), 0) + elif isinstance(deadline, datetime.timedelta): + delay = tornado.ioloop._Timeout.timedelta_to_seconds(deadline) + else: + raise TypeError("Unsupported deadline %r") + return self.reactor.callLater(delay, self._run_callback, wrap(callback)) + + def remove_timeout(self, timeout): + if timeout.active(): + timeout.cancel() + + def add_callback(self, callback, *args, **kwargs): + self.reactor.callFromThread(self._run_callback, + wrap(callback), *args, **kwargs) + + def add_callback_from_signal(self, callback, *args, **kwargs): + self.add_callback(callback, *args, **kwargs) + + +class TwistedResolver(Resolver): + """Twisted-based asynchronous resolver. + + This is a non-blocking and non-threaded resolver. It is + recommended only when threads cannot be used, since it has + limitations compared to the standard ``getaddrinfo``-based + `~tornado.netutil.Resolver` and + `~tornado.netutil.ThreadedResolver`. Specifically, it returns at + most one result, and arguments other than ``host`` and ``family`` + are ignored. It may fail to resolve when ``family`` is not + ``socket.AF_UNSPEC``. + + Requires Twisted 12.1 or newer. + """ + def initialize(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + # partial copy of twisted.names.client.createResolver, which doesn't + # allow for a reactor to be passed in. + self.reactor = tornado.platform.twisted.TornadoReactor(io_loop) + + host_resolver = twisted.names.hosts.Resolver('/etc/hosts') + cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor) + real_resolver = twisted.names.client.Resolver('/etc/resolv.conf', + reactor=self.reactor) + self.resolver = twisted.names.resolve.ResolverChain( + [host_resolver, cache_resolver, real_resolver]) + + @gen.coroutine + def resolve(self, host, port, family=0): + # getHostByName doesn't accept IP addresses, so if the input + # looks like an IP address just return it immediately. + if twisted.internet.abstract.isIPAddress(host): + resolved = host + resolved_family = socket.AF_INET + elif twisted.internet.abstract.isIPv6Address(host): + resolved = host + resolved_family = socket.AF_INET6 + else: + deferred = self.resolver.getHostByName(utf8(host)) + resolved = yield gen.Task(deferred.addCallback) + if twisted.internet.abstract.isIPAddress(resolved): + resolved_family = socket.AF_INET + elif twisted.internet.abstract.isIPv6Address(resolved): + resolved_family = socket.AF_INET6 + else: + resolved_family = socket.AF_UNSPEC + if family != socket.AF_UNSPEC and family != resolved_family: + raise Exception('Requested socket family %d but got %d' % + (family, resolved_family)) + result = [ + (resolved_family, (resolved, port)), + ] + raise gen.Return(result) diff -Nru python-tornado-2.1.0/tornado/platform/windows.py python-tornado-3.1.1/tornado/platform/windows.py --- python-tornado-2.1.0/tornado/platform/windows.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/platform/windows.py 2013-08-04 19:34:21.000000000 +0000 @@ -1,13 +1,10 @@ # NOTE: win32 support is currently experimental, and not recommended # for production use. + +from __future__ import absolute_import, division, print_function, with_statement import ctypes import ctypes.wintypes -import socket -import errno - -from tornado.platform import interface -from tornado.util import b # See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation @@ -21,77 +18,3 @@ success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0) if not success: raise ctypes.GetLastError() - - -class Waker(interface.Waker): - """Create an OS independent asynchronous pipe""" - def __init__(self): - # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py - - self.writer = socket.socket() - # Disable buffering -- pulling the trigger sends 1 byte, - # and we want that sent immediately, to wake up ASAP. - self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - count = 0 - while 1: - count += 1 - # Bind to a local port; for efficiency, let the OS pick - # a free port for us. - # Unfortunately, stress tests showed that we may not - # be able to connect to that port ("Address already in - # use") despite that the OS picked it. This appears - # to be a race bug in the Windows socket implementation. - # So we loop until a connect() succeeds (almost always - # on the first try). See the long thread at - # http://mail.zope.org/pipermail/zope/2005-July/160433.html - # for hideous details. - a = socket.socket() - a.bind(("127.0.0.1", 0)) - connect_address = a.getsockname() # assigned (host, port) pair - a.listen(1) - try: - self.writer.connect(connect_address) - break # success - except socket.error, detail: - if detail[0] != errno.WSAEADDRINUSE: - # "Address already in use" is the only error - # I've seen on two WinXP Pro SP2 boxes, under - # Pythons 2.3.5 and 2.4.1. - raise - # (10048, 'Address already in use') - # assert count <= 2 # never triggered in Tim's tests - if count >= 10: # I've never seen it go above 2 - a.close() - self.writer.close() - raise socket.error("Cannot bind trigger!") - # Close `a` and try again. Note: I originally put a short - # sleep() here, but it didn't appear to help or hurt. - a.close() - - self.reader, addr = a.accept() - self.reader.setblocking(0) - self.writer.setblocking(0) - a.close() - self.reader_fd = self.reader.fileno() - - def fileno(self): - return self.reader.fileno() - - def wake(self): - try: - self.writer.send(b("x")) - except IOError: - pass - - def consume(self): - try: - while True: - result = self.reader.recv(1024) - if not result: break - except IOError: - pass - - def close(self): - self.reader.close() - self.writer.close() diff -Nru python-tornado-2.1.0/tornado/process.py python-tornado-3.1.1/tornado/process.py --- python-tornado-2.1.0/tornado/process.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/process.py 2013-09-01 18:41:35.000000000 +0000 @@ -14,37 +14,48 @@ # License for the specific language governing permissions and limitations # under the License. -"""Utilities for working with multiple processes.""" +"""Utilities for working with multiple processes, including both forking +the server into multiple processes and managing subprocesses. +""" + +from __future__ import absolute_import, division, print_function, with_statement import errno -import logging +import multiprocessing import os +import signal +import subprocess import sys import time from binascii import hexlify from tornado import ioloop +from tornado.iostream import PipeIOStream +from tornado.log import gen_log +from tornado.platform.auto import set_close_exec +from tornado import stack_context try: - import multiprocessing # Python 2.6+ -except ImportError: - multiprocessing = None + long # py2 +except NameError: + long = int # py3 + def cpu_count(): """Returns the number of processors on this machine.""" - if multiprocessing is not None: - try: - return multiprocessing.cpu_count() - except NotImplementedError: - pass + try: + return multiprocessing.cpu_count() + except NotImplementedError: + pass try: return os.sysconf("SC_NPROCESSORS_CONF") except ValueError: pass - logging.error("Could not detect number of processors; assuming 1") + gen_log.error("Could not detect number of processors; assuming 1") return 1 + def _reseed_random(): if 'random' not in sys.modules: return @@ -55,12 +66,20 @@ try: seed = long(hexlify(os.urandom(16)), 16) except NotImplementedError: - seed(int(time.time() * 1000) ^ os.getpid()) + seed = int(time.time() * 1000) ^ os.getpid() random.seed(seed) +def _pipe_cloexec(): + r, w = os.pipe() + set_close_exec(r) + set_close_exec(w) + return r, w + + _task_id = None + def fork_processes(num_processes, max_restarts=100): """Starts multiple worker processes. @@ -93,8 +112,9 @@ raise RuntimeError("Cannot run in multiple processes: IOLoop instance " "has already been initialized. You cannot call " "IOLoop.instance() before calling start_processes()") - logging.info("Starting %d processes", num_processes) + gen_log.info("Starting %d processes", num_processes) children = {} + def start_child(i): pid = os.fork() if pid == 0: @@ -108,12 +128,13 @@ return None for i in range(num_processes): id = start_child(i) - if id is not None: return id + if id is not None: + return id num_restarts = 0 while children: try: pid, status = os.wait() - except OSError, e: + except OSError as e: if e.errno == errno.EINTR: continue raise @@ -121,19 +142,26 @@ continue id = children.pop(pid) if os.WIFSIGNALED(status): - logging.warning("child %d (pid %d) killed by signal %d, restarting", + gen_log.warning("child %d (pid %d) killed by signal %d, restarting", id, pid, os.WTERMSIG(status)) elif os.WEXITSTATUS(status) != 0: - logging.warning("child %d (pid %d) exited with status %d, restarting", + gen_log.warning("child %d (pid %d) exited with status %d, restarting", id, pid, os.WEXITSTATUS(status)) else: - logging.info("child %d (pid %d) exited normally", id, pid) + gen_log.info("child %d (pid %d) exited normally", id, pid) continue num_restarts += 1 if num_restarts > max_restarts: raise RuntimeError("Too many child restarts, giving up") new_id = start_child(id) - if new_id is not None: return new_id + if new_id is not None: + return new_id + # All child processes exited cleanly, so exit the master process + # instead of just returning to right after the call to + # fork_processes (which will probably just start up another IOLoop + # unless the caller checks the return value). + sys.exit(0) + def task_id(): """Returns the current task id, if any. @@ -142,3 +170,123 @@ """ global _task_id return _task_id + + +class Subprocess(object): + """Wraps ``subprocess.Popen`` with IOStream support. + + The constructor is the same as ``subprocess.Popen`` with the following + additions: + + * ``stdin``, ``stdout``, and ``stderr`` may have the value + ``tornado.process.Subprocess.STREAM``, which will make the corresponding + attribute of the resulting Subprocess a `.PipeIOStream`. + * A new keyword argument ``io_loop`` may be used to pass in an IOLoop. + """ + STREAM = object() + + _initialized = False + _waiting = {} + + def __init__(self, *args, **kwargs): + self.io_loop = kwargs.pop('io_loop', None) or ioloop.IOLoop.current() + to_close = [] + if kwargs.get('stdin') is Subprocess.STREAM: + in_r, in_w = _pipe_cloexec() + kwargs['stdin'] = in_r + to_close.append(in_r) + self.stdin = PipeIOStream(in_w, io_loop=self.io_loop) + if kwargs.get('stdout') is Subprocess.STREAM: + out_r, out_w = _pipe_cloexec() + kwargs['stdout'] = out_w + to_close.append(out_w) + self.stdout = PipeIOStream(out_r, io_loop=self.io_loop) + if kwargs.get('stderr') is Subprocess.STREAM: + err_r, err_w = _pipe_cloexec() + kwargs['stderr'] = err_w + to_close.append(err_w) + self.stderr = PipeIOStream(err_r, io_loop=self.io_loop) + self.proc = subprocess.Popen(*args, **kwargs) + for fd in to_close: + os.close(fd) + for attr in ['stdin', 'stdout', 'stderr', 'pid']: + if not hasattr(self, attr): # don't clobber streams set above + setattr(self, attr, getattr(self.proc, attr)) + self._exit_callback = None + self.returncode = None + + def set_exit_callback(self, callback): + """Runs ``callback`` when this process exits. + + The callback takes one argument, the return code of the process. + + This method uses a ``SIGCHILD`` handler, which is a global setting + and may conflict if you have other libraries trying to handle the + same signal. If you are using more than one ``IOLoop`` it may + be necessary to call `Subprocess.initialize` first to designate + one ``IOLoop`` to run the signal handlers. + + In many cases a close callback on the stdout or stderr streams + can be used as an alternative to an exit callback if the + signal handler is causing a problem. + """ + self._exit_callback = stack_context.wrap(callback) + Subprocess.initialize(self.io_loop) + Subprocess._waiting[self.pid] = self + Subprocess._try_cleanup_process(self.pid) + + @classmethod + def initialize(cls, io_loop=None): + """Initializes the ``SIGCHILD`` handler. + + The signal handler is run on an `.IOLoop` to avoid locking issues. + Note that the `.IOLoop` used for signal handling need not be the + same one used by individual Subprocess objects (as long as the + ``IOLoops`` are each running in separate threads). + """ + if cls._initialized: + return + if io_loop is None: + io_loop = ioloop.IOLoop.current() + cls._old_sigchld = signal.signal( + signal.SIGCHLD, + lambda sig, frame: io_loop.add_callback_from_signal(cls._cleanup)) + cls._initialized = True + + @classmethod + def uninitialize(cls): + """Removes the ``SIGCHILD`` handler.""" + if not cls._initialized: + return + signal.signal(signal.SIGCHLD, cls._old_sigchld) + cls._initialized = False + + @classmethod + def _cleanup(cls): + for pid in list(cls._waiting.keys()): # make a copy + cls._try_cleanup_process(pid) + + @classmethod + def _try_cleanup_process(cls, pid): + try: + ret_pid, status = os.waitpid(pid, os.WNOHANG) + except OSError as e: + if e.args[0] == errno.ECHILD: + return + if ret_pid == 0: + return + assert ret_pid == pid + subproc = cls._waiting.pop(pid) + subproc.io_loop.add_callback_from_signal( + subproc._set_returncode, status) + + def _set_returncode(self, status): + if os.WIFSIGNALED(status): + self.returncode = -os.WTERMSIG(status) + else: + assert os.WIFEXITED(status) + self.returncode = os.WEXITSTATUS(status) + if self._exit_callback: + callback = self._exit_callback + self._exit_callback = None + callback(self.returncode) diff -Nru python-tornado-2.1.0/tornado/simple_httpclient.py python-tornado-3.1.1/tornado/simple_httpclient.py --- python-tornado-2.1.0/tornado/simple_httpclient.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/simple_httpclient.py 2013-08-04 19:34:21.000000000 +0000 @@ -1,25 +1,24 @@ #!/usr/bin/env python -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, with_statement from tornado.escape import utf8, _unicode, native_str -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy from tornado.httputil import HTTPHeaders from tornado.iostream import IOStream, SSLIOStream +from tornado.netutil import Resolver, OverrideResolver +from tornado.log import gen_log from tornado import stack_context -from tornado.util import b +from tornado.util import GzipDecompressor import base64 import collections -import contextlib import copy import functools -import logging import os.path import re import socket -import time -import urlparse -import zlib +import ssl +import sys try: from io import BytesIO # python 3 @@ -27,52 +26,38 @@ from cStringIO import StringIO as BytesIO # python 2 try: - import ssl # python 2.6+ + import urlparse # py2 except ImportError: - ssl = None + import urllib.parse as urlparse # py3 _DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' + class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. It does not currently implement all applicable parts of the HTTP - specification, but it does enough to work with major web service APIs - (mostly tested against the Twitter API so far). - - This class has not been tested extensively in production and - should be considered somewhat experimental as of the release of - tornado 1.2. It is intended to become the default AsyncHTTPClient - implementation in a future release. It may either be used - directly, or to facilitate testing of this class with an existing - application, setting the environment variable - USE_SIMPLE_HTTPCLIENT=1 will cause this class to transparently - replace tornado.httpclient.AsyncHTTPClient. + specification, but it does enough to work with major web service APIs. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections - are not reused, and callers cannot select the network interface to be + are not reused, and callers cannot select the network interface to be used. - - Python 2.6 or higher is required for HTTPS support. Users of Python 2.5 - should use the curl-based AsyncHTTPClient if HTTPS support is required. - """ - def initialize(self, io_loop=None, max_clients=10, - max_simultaneous_connections=None, - hostname_mapping=None, max_buffer_size=104857600): + def initialize(self, io_loop, max_clients=10, + hostname_mapping=None, max_buffer_size=104857600, + resolver=None, defaults=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. - max_clients is the number of concurrent requests that can be in - progress. max_simultaneous_connections has no effect and is accepted - only for compatibility with the curl-based AsyncHTTPClient. Note - that these arguments are only used when the client is first created, - and will be ignored when an existing client is reused. + max_clients is the number of concurrent requests that can be + in progress. Note that this arguments are only used when the + client is first created, and will be ignored when an existing + client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide @@ -82,25 +67,34 @@ max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ - self.io_loop = io_loop + super(SimpleAsyncHTTPClient, self).initialize(io_loop, + defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} - self.hostname_mapping = hostname_mapping self.max_buffer_size = max_buffer_size + if resolver: + self.resolver = resolver + self.own_resolver = False + else: + self.resolver = Resolver(io_loop=io_loop) + self.own_resolver = True + if hostname_mapping is not None: + self.resolver = OverrideResolver(resolver=self.resolver, + mapping=hostname_mapping) + + def close(self): + super(SimpleAsyncHTTPClient, self).close() + if self.own_resolver: + self.resolver.close() - def fetch(self, request, callback, **kwargs): - if not isinstance(request, HTTPRequest): - request = HTTPRequest(url=request, **kwargs) - if not isinstance(request.headers, HTTPHeaders): - request.headers = HTTPHeaders(request.headers) - callback = stack_context.wrap(callback) + def fetch_impl(self, request, callback): self.queue.append((request, callback)) self._process_queue() if self.queue: - logging.debug("max_clients limit reached, request queued. " + gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( - len(self.active), len(self.queue))) + len(self.active), len(self.queue))) def _process_queue(self): with stack_context.NullContext(): @@ -108,45 +102,45 @@ request, callback = self.queue.popleft() key = object() self.active[key] = (request, callback) - _HTTPConnection(self.io_loop, self, request, - functools.partial(self._release_fetch, key), - callback, - self.max_buffer_size) + release_callback = functools.partial(self._release_fetch, key) + self._handle_request(request, release_callback, callback) + + def _handle_request(self, request, release_callback, final_callback): + _HTTPConnection(self.io_loop, self, request, release_callback, + final_callback, self.max_buffer_size, self.resolver) def _release_fetch(self, key): del self.active[key] self._process_queue() - class _HTTPConnection(object): - _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"]) + _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) def __init__(self, io_loop, client, request, release_callback, - final_callback, max_buffer_size): - self.start_time = time.time() + final_callback, max_buffer_size, resolver): + self.start_time = io_loop.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback + self.max_buffer_size = max_buffer_size + self.resolver = resolver self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None - with stack_context.StackContext(self.cleanup): - parsed = urlparse.urlsplit(_unicode(self.request.url)) - if ssl is None and parsed.scheme == "https": - raise ValueError("HTTPS requires either python2.6+ or " - "curl_httpclient") - if parsed.scheme not in ("http", "https"): + with stack_context.ExceptionStackContext(self._handle_exception): + self.parsed = urlparse.urlsplit(_unicode(self.request.url)) + if self.parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. - netloc = parsed.netloc + netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) @@ -155,12 +149,11 @@ port = int(match.group(2)) else: host = netloc - port = 443 if parsed.scheme == "https" else 80 + port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] - if self.client.hostname_mapping is not None: - host = self.client.hostname_mapping.get(host, host) + self.parsed_hostname = host # save final host for _on_connect if request.allow_ipv6: af = socket.AF_UNSPEC @@ -169,106 +162,139 @@ # so restrict to ipv4 by default. af = socket.AF_INET - addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, - 0, 0) - af, socktype, proto, canonname, sockaddr = addrinfo[0] - - if parsed.scheme == "https": - ssl_options = {} - if request.validate_cert: - ssl_options["cert_reqs"] = ssl.CERT_REQUIRED - if request.ca_certs is not None: - ssl_options["ca_certs"] = request.ca_certs - else: - ssl_options["ca_certs"] = _DEFAULT_CA_CERTS - if request.client_key is not None: - ssl_options["keyfile"] = request.client_key - if request.client_cert is not None: - ssl_options["certfile"] = request.client_cert - self.stream = SSLIOStream(socket.socket(af, socktype, proto), - io_loop=self.io_loop, - ssl_options=ssl_options, - max_buffer_size=max_buffer_size) + self.resolver.resolve(host, port, af, callback=self._on_resolve) + + def _on_resolve(self, addrinfo): + self.stream = self._create_stream(addrinfo) + timeout = min(self.request.connect_timeout, self.request.request_timeout) + if timeout: + self._timeout = self.io_loop.add_timeout( + self.start_time + timeout, + stack_context.wrap(self._on_timeout)) + self.stream.set_close_callback(self._on_close) + # ipv6 addresses are broken (in self.parsed.hostname) until + # 2.7, here is correctly parsed value calculated in __init__ + sockaddr = addrinfo[0][1] + self.stream.connect(sockaddr, self._on_connect, + server_hostname=self.parsed_hostname) + + def _create_stream(self, addrinfo): + af = addrinfo[0][0] + if self.parsed.scheme == "https": + ssl_options = {} + if self.request.validate_cert: + ssl_options["cert_reqs"] = ssl.CERT_REQUIRED + if self.request.ca_certs is not None: + ssl_options["ca_certs"] = self.request.ca_certs else: - self.stream = IOStream(socket.socket(af, socktype, proto), - io_loop=self.io_loop, - max_buffer_size=max_buffer_size) - timeout = min(request.connect_timeout, request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - self._on_timeout) - self.stream.set_close_callback(self._on_close) - self.stream.connect(sockaddr, - functools.partial(self._on_connect, parsed)) + ssl_options["ca_certs"] = _DEFAULT_CA_CERTS + if self.request.client_key is not None: + ssl_options["keyfile"] = self.request.client_key + if self.request.client_cert is not None: + ssl_options["certfile"] = self.request.client_cert + + # SSL interoperability is tricky. We want to disable + # SSLv2 for security reasons; it wasn't disabled by default + # until openssl 1.0. The best way to do this is to use + # the SSL_OP_NO_SSLv2, but that wasn't exposed to python + # until 3.2. Python 2.7 adds the ciphers argument, which + # can also be used to disable SSLv2. As a last resort + # on python 2.6, we set ssl_version to SSLv3. This is + # more narrow than we'd like since it also breaks + # compatibility with servers configured for TLSv1 only, + # but nearly all servers support SSLv3: + # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html + if sys.version_info >= (2, 7): + ssl_options["ciphers"] = "DEFAULT:!SSLv2" + else: + # This is really only necessary for pre-1.0 versions + # of openssl, but python 2.6 doesn't expose version + # information. + ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 + + return SSLIOStream(socket.socket(af), + io_loop=self.io_loop, + ssl_options=ssl_options, + max_buffer_size=self.max_buffer_size) + else: + return IOStream(socket.socket(af), + io_loop=self.io_loop, + max_buffer_size=self.max_buffer_size) def _on_timeout(self): self._timeout = None - self._run_callback(HTTPResponse(self.request, 599, - request_time=time.time() - self.start_time, - error=HTTPError(599, "Timeout"))) - self.stream.close() + if self.final_callback is not None: + raise HTTPError(599, "Timeout") - def _on_connect(self, parsed): + def _remove_timeout(self): if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None + + def _on_connect(self): + self._remove_timeout() if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, - self._on_timeout) - if (self.request.validate_cert and - isinstance(self.stream, SSLIOStream)): - match_hostname(self.stream.socket.getpeercert(), - parsed.hostname) + stack_context.wrap(self._on_timeout)) if (self.request.method not in self._SUPPORTED_METHODS and - not self.request.allow_nonstandard_methods): + not self.request.allow_nonstandard_methods): raise KeyError("unknown method %s" % self.request.method) for key in ('network_interface', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'): if getattr(self.request, key, None): raise NotImplementedError('%s not supported' % key) + if "Connection" not in self.request.headers: + self.request.headers["Connection"] = "close" if "Host" not in self.request.headers: - self.request.headers["Host"] = parsed.netloc + if '@' in self.parsed.netloc: + self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1] + else: + self.request.headers["Host"] = self.parsed.netloc username, password = None, None - if parsed.username is not None: - username, password = parsed.username, parsed.password + if self.parsed.username is not None: + username, password = self.parsed.username, self.parsed.password elif self.request.auth_username is not None: username = self.request.auth_username - password = self.request.auth_password + password = self.request.auth_password or '' if username is not None: - auth = utf8(username) + b(":") + utf8(password) - self.request.headers["Authorization"] = (b("Basic ") + + if self.request.auth_mode not in (None, "basic"): + raise ValueError("unsupported auth_mode %s", + self.request.auth_mode) + auth = utf8(username) + b":" + utf8(password) + self.request.headers["Authorization"] = (b"Basic " + base64.b64encode(auth)) if self.request.user_agent: self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: - if self.request.method in ("POST", "PUT"): + if self.request.method in ("POST", "PATCH", "PUT"): assert self.request.body is not None else: assert self.request.body is None if self.request.body is not None: self.request.headers["Content-Length"] = str(len( - self.request.body)) + self.request.body)) if (self.request.method == "POST" and - "Content-Type" not in self.request.headers): + "Content-Type" not in self.request.headers): self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" if self.request.use_gzip: self.request.headers["Accept-Encoding"] = "gzip" - req_path = ((parsed.path or '/') + - (('?' + parsed.query) if parsed.query else '')) + req_path = ((self.parsed.path or '/') + + (('?' + self.parsed.query) if self.parsed.query else '')) request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method, req_path))] for k, v in self.request.headers.get_all(): - line = utf8(k) + b(": ") + utf8(v) - if b('\n') in line: + line = utf8(k) + b": " + utf8(v) + if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) request_lines.append(line) - self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n")) + request_str = b"\r\n".join(request_lines) + b"\r\n\r\n" if self.request.body is not None: - self.stream.write(self.request.body) - self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers) + request_str += self.request.body + self.stream.set_nodelay(True) + self.stream.write(request_str) + self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers) def _release(self): if self.release_callback is not None: @@ -281,110 +307,181 @@ if self.final_callback is not None: final_callback = self.final_callback self.final_callback = None - final_callback(response) + self.io_loop.add_callback(final_callback, response) - @contextlib.contextmanager - def cleanup(self): - try: - yield - except Exception, e: - logging.warning("uncaught exception", exc_info=True) - self._run_callback(HTTPResponse(self.request, 599, error=e, - request_time=time.time() - self.start_time, - )) + def _handle_exception(self, typ, value, tb): + if self.final_callback: + self._remove_timeout() + self._run_callback(HTTPResponse(self.request, 599, error=value, + request_time=self.io_loop.time() - self.start_time, + )) + + if hasattr(self, "stream"): + self.stream.close() + return True + else: + # If our callback has already been called, we are probably + # catching an exception that is not caused by us but rather + # some child of our callback. Rather than drop it on the floor, + # pass it along. + return False def _on_close(self): - self._run_callback(HTTPResponse( - self.request, 599, - request_time=time.time() - self.start_time, - error=HTTPError(599, "Connection closed"))) + if self.final_callback is not None: + message = "Connection closed" + if self.stream.error: + message = str(self.stream.error) + raise HTTPError(599, message) + + def _handle_1xx(self, code): + self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers) def _on_headers(self, data): data = native_str(data.decode("latin1")) first_line, _, header_data = data.partition("\n") - match = re.match("HTTP/1.[01] ([0-9]+)", first_line) + match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line) assert match - self.code = int(match.group(1)) + code = int(match.group(1)) self.headers = HTTPHeaders.parse(header_data) - if self.request.header_callback is not None: - for k, v in self.headers.get_all(): - self.request.header_callback("%s: %s\r\n" % (k, v)) - if (self.request.use_gzip and - self.headers.get("Content-Encoding") == "gzip"): - # Magic parameter makes zlib module understand gzip header - # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib - self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS) - if self.headers.get("Transfer-Encoding") == "chunked": - self.chunks = [] - self.stream.read_until(b("\r\n"), self._on_chunk_length) - elif "Content-Length" in self.headers: + if 100 <= code < 200: + self._handle_1xx(code) + return + else: + self.code = code + self.reason = match.group(2) + + if "Content-Length" in self.headers: if "," in self.headers["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', self.headers["Content-Length"]) if any(i != pieces[0] for i in pieces): - raise ValueError("Multiple unequal Content-Lengths: %r" % + raise ValueError("Multiple unequal Content-Lengths: %r" % self.headers["Content-Length"]) self.headers["Content-Length"] = pieces[0] - self.stream.read_bytes(int(self.headers["Content-Length"]), - self._on_body) + content_length = int(self.headers["Content-Length"]) + else: + content_length = None + + if self.request.header_callback is not None: + # re-attach the newline we split on earlier + self.request.header_callback(first_line + _) + for k, v in self.headers.get_all(): + self.request.header_callback("%s: %s\r\n" % (k, v)) + self.request.header_callback('\r\n') + + if self.request.method == "HEAD" or self.code == 304: + # HEAD requests and 304 responses never have content, even + # though they may have content-length headers + self._on_body(b"") + return + if 100 <= self.code < 200 or self.code == 204: + # These response codes never have bodies + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 + if ("Transfer-Encoding" in self.headers or + content_length not in (None, 0)): + raise ValueError("Response with code %d should not have body" % + self.code) + self._on_body(b"") + return + + if (self.request.use_gzip and + self.headers.get("Content-Encoding") == "gzip"): + self._decompressor = GzipDecompressor() + if self.headers.get("Transfer-Encoding") == "chunked": + self.chunks = [] + self.stream.read_until(b"\r\n", self._on_chunk_length) + elif content_length is not None: + self.stream.read_bytes(content_length, self._on_body) else: self.stream.read_until_close(self._on_body) def _on_body(self, data): - if self._timeout is not None: - self.io_loop.remove_timeout(self._timeout) - self._timeout = None - if self._decompressor: - data = self._decompressor.decompress(data) - if self.request.streaming_callback: - if self.chunks is None: - # if chunks is not None, we already called streaming_callback - # in _on_chunk_data - self.request.streaming_callback(data) - buffer = BytesIO() - else: - buffer = BytesIO(data) # TODO: don't require one big string? + self._remove_timeout() original_request = getattr(self.request, "original_request", self.request) if (self.request.follow_redirects and self.request.max_redirects > 0 and - self.code in (301, 302)): - new_request = copy.copy(self.request) + self.code in (301, 302, 303, 307)): + assert isinstance(self.request, _RequestProxy) + new_request = copy.copy(self.request.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) - new_request.max_redirects -= 1 + new_request.max_redirects = self.request.max_redirects - 1 del new_request.headers["Host"] + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + # Client SHOULD make a GET request after a 303. + # According to the spec, 302 should be followed by the same + # method as the original request, but in practice browsers + # treat 302 the same as 303, and many servers use 302 for + # compatibility with pre-HTTP/1.1 user agents which don't + # understand the 303 status. + if self.code in (302, 303): + new_request.method = "GET" + new_request.body = None + for h in ["Content-Length", "Content-Type", + "Content-Encoding", "Transfer-Encoding"]: + try: + del self.request.headers[h] + except KeyError: + pass new_request.original_request = original_request final_callback = self.final_callback self.final_callback = None self._release() self.client.fetch(new_request, final_callback) - self.stream.close() + self._on_end_request() return + if self._decompressor: + data = (self._decompressor.decompress(data) + + self._decompressor.flush()) + if self.request.streaming_callback: + if self.chunks is None: + # if chunks is not None, we already called streaming_callback + # in _on_chunk_data + self.request.streaming_callback(data) + buffer = BytesIO() + else: + buffer = BytesIO(data) # TODO: don't require one big string? response = HTTPResponse(original_request, - self.code, headers=self.headers, - request_time=time.time() - self.start_time, + self.code, reason=self.reason, + headers=self.headers, + request_time=self.io_loop.time() - self.start_time, buffer=buffer, effective_url=self.request.url) self._run_callback(response) + self._on_end_request() + + def _on_end_request(self): self.stream.close() def _on_chunk_length(self, data): # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 length = int(data.strip(), 16) if length == 0: - # all the data has been decompressed, so we don't need to - # decompress again in _on_body - self._decompressor = None - self._on_body(b('').join(self.chunks)) + if self._decompressor is not None: + tail = self._decompressor.flush() + if tail: + # I believe the tail will always be empty (i.e. + # decompress will return all it can). The purpose + # of the flush call is to detect errors such + # as truncated input. But in case it ever returns + # anything, treat it as an extra chunk + if self.request.streaming_callback is not None: + self.request.streaming_callback(tail) + else: + self.chunks.append(tail) + # all the data has been decompressed, so we don't need to + # decompress again in _on_body + self._decompressor = None + self._on_body(b''.join(self.chunks)) else: self.stream.read_bytes(length + 2, # chunk ends with \r\n - self._on_chunk_data) + self._on_chunk_data) def _on_chunk_data(self, data): - assert data[-2:] == b("\r\n") + assert data[-2:] == b"\r\n" chunk = data[:-2] if self._decompressor: chunk = self._decompressor.decompress(chunk) @@ -392,67 +489,9 @@ self.request.streaming_callback(chunk) else: self.chunks.append(chunk) - self.stream.read_until(b("\r\n"), self._on_chunk_length) + self.stream.read_until(b"\r\n", self._on_chunk_length) -# match_hostname was added to the standard library ssl module in python 3.2. -# The following code was backported for older releases and copied from -# https://bitbucket.org/brandon/backports.ssl_match_hostname -class CertificateError(ValueError): - pass - -def _dnsname_to_pat(dn): - pats = [] - for frag in dn.split(r'.'): - if frag == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - else: - # Otherwise, '*' matches any dotless fragment. - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - -def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules - are mostly followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not san: - # The subject is only checked when subjectAltName is empty - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") - if __name__ == "__main__": AsyncHTTPClient.configure(SimpleAsyncHTTPClient) main() diff -Nru python-tornado-2.1.0/tornado/stack_context.py python-tornado-3.1.1/tornado/stack_context.py --- python-tornado-2.1.0/tornado/stack_context.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/stack_context.py 2013-08-04 19:34:21.000000000 +0000 @@ -14,20 +14,21 @@ # License for the specific language governing permissions and limitations # under the License. -'''StackContext allows applications to maintain threadlocal-like state +"""`StackContext` allows applications to maintain threadlocal-like state that follows execution as it moves to other execution contexts. The motivating examples are to eliminate the need for explicit -async_callback wrappers (as in tornado.web.RequestHandler), and to +``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to allow some additional context to be kept for logging. -This is slightly magic, but it's an extension of the idea that an exception -handler is a kind of stack-local state and when that stack is suspended -and resumed in a new context that state needs to be preserved. StackContext -shifts the burden of restoring that state from each call site (e.g. -wrapping each AsyncHTTPClient callback in async_callback) to the mechanisms -that transfer control from one context to another (e.g. AsyncHTTPClient -itself, IOLoop, thread pools, etc). +This is slightly magic, but it's an extension of the idea that an +exception handler is a kind of stack-local state and when that stack +is suspended and resumed in a new context that state needs to be +preserved. `StackContext` shifts the burden of restoring that state +from each call site (e.g. wrapping each `.AsyncHTTPClient` callback +in ``async_callback``) to the mechanisms that transfer control from +one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, +thread pools, etc). Example usage:: @@ -45,23 +46,47 @@ # in the ioloop. http_client.fetch(url, callback) ioloop.start() -''' -from __future__ import with_statement +Most applications shouln't have to work with `StackContext` directly. +Here are a few rules of thumb for when it's necessary: + +* If you're writing an asynchronous library that doesn't rely on a + stack_context-aware library like `tornado.ioloop` or `tornado.iostream` + (for example, if you're writing a thread pool), use + `.stack_context.wrap()` before any asynchronous operations to capture the + stack context from where the operation was started. + +* If you're writing an asynchronous library that has some shared + resources (such as a connection pool), create those shared resources + within a ``with stack_context.NullContext():`` block. This will prevent + ``StackContexts`` from leaking from one request to another. + +* If you want to write something like an exception handler that will + persist across asynchronous calls, create a new `StackContext` (or + `ExceptionStackContext`), and make your asynchronous calls in a ``with`` + block that references your `StackContext`. +""" + +from __future__ import absolute_import, division, print_function, with_statement -import contextlib -import functools -import itertools import sys import threading +from tornado.util import raise_exc_info + + +class StackContextInconsistentError(Exception): + pass + + class _State(threading.local): def __init__(self): - self.contexts = () + self.contexts = (tuple(), None) _state = _State() + class StackContext(object): - '''Establishes the given context as a StackContext that will be transferred. + """Establishes the given context as a StackContext that will be transferred. Note that the parameter is a callable that returns a context manager, not the context itself. That is, where for a @@ -72,154 +97,280 @@ StackContext takes the function itself rather than its result:: with StackContext(my_context): - ''' + + The result of ``with StackContext() as cb:`` is a deactivation + callback. Run this callback when the StackContext is no longer + needed to ensure that it is not propagated any further (note that + deactivating a context does not affect any instances of that + context that are currently pending). This is an advanced feature + and not necessary in most applications. + """ def __init__(self, context_factory): self.context_factory = context_factory + self.contexts = [] + self.active = True + + def _deactivate(self): + self.active = False + + # StackContext protocol + def enter(self): + context = self.context_factory() + self.contexts.append(context) + context.__enter__() + + def exit(self, type, value, traceback): + context = self.contexts.pop() + context.__exit__(type, value, traceback) # Note that some of this code is duplicated in ExceptionStackContext # below. ExceptionStackContext is more common and doesn't need # the full generality of this class. def __enter__(self): self.old_contexts = _state.contexts - # _state.contexts is a tuple of (class, arg) pairs - _state.contexts = (self.old_contexts + - ((StackContext, self.context_factory),)) + self.new_contexts = (self.old_contexts[0] + (self,), self) + _state.contexts = self.new_contexts + try: - self.context = self.context_factory() - self.context.__enter__() - except Exception: + self.enter() + except: _state.contexts = self.old_contexts raise + return self._deactivate + def __exit__(self, type, value, traceback): try: - return self.context.__exit__(type, value, traceback) + self.exit(type, value, traceback) finally: + final_contexts = _state.contexts _state.contexts = self.old_contexts + # Generator coroutines and with-statements with non-local + # effects interact badly. Check here for signs of + # the stack getting out of sync. + # Note that this check comes after restoring _state.context + # so that if it fails things are left in a (relatively) + # consistent state. + if final_contexts is not self.new_contexts: + raise StackContextInconsistentError( + 'stack_context inconsistency (may be caused by yield ' + 'within a "with StackContext" block)') + + # Break up a reference to itself to allow for faster GC on CPython. + self.new_contexts = None + + class ExceptionStackContext(object): - '''Specialization of StackContext for exception handling. + """Specialization of StackContext for exception handling. - The supplied exception_handler function will be called in the + The supplied ``exception_handler`` function will be called in the event of an uncaught exception in this context. The semantics are similar to a try/finally clause, and intended use cases are to log an error, close a socket, or similar cleanup actions. The - exc_info triple (type, value, traceback) will be passed to the + ``exc_info`` triple ``(type, value, traceback)`` will be passed to the exception_handler function. If the exception handler returns true, the exception will be consumed and will not be propagated to other exception handlers. - ''' + """ def __init__(self, exception_handler): self.exception_handler = exception_handler + self.active = True + + def _deactivate(self): + self.active = False + + def exit(self, type, value, traceback): + if type is not None: + return self.exception_handler(type, value, traceback) def __enter__(self): self.old_contexts = _state.contexts - _state.contexts = (self.old_contexts + - ((ExceptionStackContext, self.exception_handler),)) + self.new_contexts = (self.old_contexts[0], self) + _state.contexts = self.new_contexts + + return self._deactivate def __exit__(self, type, value, traceback): try: if type is not None: return self.exception_handler(type, value, traceback) finally: + final_contexts = _state.contexts _state.contexts = self.old_contexts + if final_contexts is not self.new_contexts: + raise StackContextInconsistentError( + 'stack_context inconsistency (may be caused by yield ' + 'within a "with StackContext" block)') + + # Break up a reference to itself to allow for faster GC on CPython. + self.new_contexts = None + + class NullContext(object): - '''Resets the StackContext. + """Resets the `StackContext`. - Useful when creating a shared resource on demand (e.g. an AsyncHTTPClient) - where the stack that caused the creating is not relevant to future - operations. - ''' + Useful when creating a shared resource on demand (e.g. an + `.AsyncHTTPClient`) where the stack that caused the creating is + not relevant to future operations. + """ def __enter__(self): self.old_contexts = _state.contexts - _state.contexts = () + _state.contexts = (tuple(), None) def __exit__(self, type, value, traceback): _state.contexts = self.old_contexts -class _StackContextWrapper(functools.partial): - pass + +def _remove_deactivated(contexts): + """Remove deactivated handlers from the chain""" + # Clean ctx handlers + stack_contexts = tuple([h for h in contexts[0] if h.active]) + + # Find new head + head = contexts[1] + while head is not None and not head.active: + head = head.old_contexts[1] + + # Process chain + ctx = head + while ctx is not None: + parent = ctx.old_contexts[1] + + while parent is not None: + if parent.active: + break + ctx.old_contexts = parent.old_contexts + parent = parent.old_contexts[1] + + ctx = parent + + return (stack_contexts, head) + def wrap(fn): - '''Returns a callable object that will restore the current StackContext + """Returns a callable object that will restore the current `StackContext` when executed. Use this whenever saving a callback to be executed later in a different execution context (either in a different thread or asynchronously in the same thread). - ''' - if fn is None or fn.__class__ is _StackContextWrapper: - return fn - # functools.wraps doesn't appear to work on functools.partial objects - #@functools.wraps(fn) - def wrapped(callback, contexts, *args, **kwargs): - if contexts is _state.contexts or not contexts: - callback(*args, **kwargs) - return - if not _state.contexts: - new_contexts = [cls(arg) for (cls, arg) in contexts] - # If we're moving down the stack, _state.contexts is a prefix - # of contexts. For each element of contexts not in that prefix, - # create a new StackContext object. - # If we're moving up the stack (or to an entirely different stack), - # _state.contexts will have elements not in contexts. Use - # NullContext to clear the state and then recreate from contexts. - elif (len(_state.contexts) > len(contexts) or - any(a[1] is not b[1] - for a, b in itertools.izip(_state.contexts, contexts))): - # contexts have been removed or changed, so start over - new_contexts = ([NullContext()] + - [cls(arg) for (cls,arg) in contexts]) - else: - new_contexts = [cls(arg) - for (cls, arg) in contexts[len(_state.contexts):]] - if len(new_contexts) > 1: - with _nested(*new_contexts): - callback(*args, **kwargs) - elif new_contexts: - with new_contexts[0]: - callback(*args, **kwargs) - else: - callback(*args, **kwargs) - if _state.contexts: - return _StackContextWrapper(wrapped, fn, _state.contexts) - else: - return _StackContextWrapper(fn) - -@contextlib.contextmanager -def _nested(*managers): - """Support multiple context managers in a single with-statement. - - Copied from the python 2.6 standard library. It's no longer present - in python 3 because the with statement natively supports multiple - context managers, but that doesn't help if the list of context - managers is not known until runtime. """ - exits = [] - vars = [] - exc = (None, None, None) - try: - for mgr in managers: - exit = mgr.__exit__ - enter = mgr.__enter__ - vars.append(enter()) - exits.append(exit) - yield vars - except: - exc = sys.exc_info() - finally: - while exits: - exit = exits.pop() - try: - if exit(*exc): - exc = (None, None, None) - except: - exc = sys.exc_info() - if exc != (None, None, None): - # Don't rely on sys.exc_info() still containing - # the right information. Another exception may - # have been raised and caught by an exit method - raise exc[0], exc[1], exc[2] + # Check if function is already wrapped + if fn is None or hasattr(fn, '_wrapped'): + return fn + + # Capture current stack head + # TODO: Any other better way to store contexts and update them in wrapped function? + cap_contexts = [_state.contexts] + + def wrapped(*args, **kwargs): + ret = None + try: + # Capture old state + current_state = _state.contexts + + # Remove deactivated items + cap_contexts[0] = contexts = _remove_deactivated(cap_contexts[0]) + + # Force new state + _state.contexts = contexts + + # Current exception + exc = (None, None, None) + top = None + + # Apply stack contexts + last_ctx = 0 + stack = contexts[0] + + # Apply state + for n in stack: + try: + n.enter() + last_ctx += 1 + except: + # Exception happened. Record exception info and store top-most handler + exc = sys.exc_info() + top = n.old_contexts[1] + + # Execute callback if no exception happened while restoring state + if top is None: + try: + ret = fn(*args, **kwargs) + except: + exc = sys.exc_info() + top = contexts[1] + + # If there was exception, try to handle it by going through the exception chain + if top is not None: + exc = _handle_exception(top, exc) + else: + # Otherwise take shorter path and run stack contexts in reverse order + while last_ctx > 0: + last_ctx -= 1 + c = stack[last_ctx] + + try: + c.exit(*exc) + except: + exc = sys.exc_info() + top = c.old_contexts[1] + break + else: + top = None + + # If if exception happened while unrolling, take longer exception handler path + if top is not None: + exc = _handle_exception(top, exc) + + # If exception was not handled, raise it + if exc != (None, None, None): + raise_exc_info(exc) + finally: + _state.contexts = current_state + return ret + wrapped._wrapped = True + return wrapped + + +def _handle_exception(tail, exc): + while tail is not None: + try: + if tail.exit(*exc): + exc = (None, None, None) + except: + exc = sys.exc_info() + + tail = tail.old_contexts[1] + + return exc + + +def run_with_stack_context(context, func): + """Run a coroutine ``func`` in the given `StackContext`. + + It is not safe to have a ``yield`` statement within a ``with StackContext`` + block, so it is difficult to use stack context with `.gen.coroutine`. + This helper function runs the function in the correct context while + keeping the ``yield`` and ``with`` statements syntactically separate. + + Example:: + + @gen.coroutine + def incorrect(): + with StackContext(ctx): + # ERROR: this will raise StackContextInconsistentError + yield other_coroutine() + + @gen.coroutine + def correct(): + yield run_with_stack_context(StackContext(ctx), other_coroutine) + + .. versionadded:: 3.1 + """ + with context: + return func() diff -Nru python-tornado-2.1.0/tornado/tcpserver.py python-tornado-3.1.1/tornado/tcpserver.py --- python-tornado-2.1.0/tornado/tcpserver.py 1970-01-01 00:00:00.000000000 +0000 +++ python-tornado-3.1.1/tornado/tcpserver.py 2013-08-04 19:34:21.000000000 +0000 @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +# Copyright 2011 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A non-blocking, single-threaded TCP server.""" +from __future__ import absolute_import, division, print_function, with_statement + +import errno +import os +import socket +import ssl + +from tornado.log import app_log +from tornado.ioloop import IOLoop +from tornado.iostream import IOStream, SSLIOStream +from tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket +from tornado import process + + +class TCPServer(object): + r"""A non-blocking, single-threaded TCP server. + + To use `TCPServer`, define a subclass which overrides the `handle_stream` + method. + + To make this server serve SSL traffic, send the ssl_options dictionary + argument with the arguments required for the `ssl.wrap_socket` method, + including "certfile" and "keyfile":: + + TCPServer(ssl_options={ + "certfile": os.path.join(data_dir, "mydomain.crt"), + "keyfile": os.path.join(data_dir, "mydomain.key"), + }) + + `TCPServer` initialization follows one of three patterns: + + 1. `listen`: simple single-process:: + + server = TCPServer() + server.listen(8888) + IOLoop.instance().start() + + 2. `bind`/`start`: simple multi-process:: + + server = TCPServer() + server.bind(8888) + server.start(0) # Forks multiple sub-processes + IOLoop.instance().start() + + When using this interface, an `.IOLoop` must *not* be passed + to the `TCPServer` constructor. `start` will always start + the server on the default singleton `.IOLoop`. + + 3. `add_sockets`: advanced multi-process:: + + sockets = bind_sockets(8888) + tornado.process.fork_processes(0) + server = TCPServer() + server.add_sockets(sockets) + IOLoop.instance().start() + + The `add_sockets` interface is more complicated, but it can be + used with `tornado.process.fork_processes` to give you more + flexibility in when the fork happens. `add_sockets` can + also be used in single-process servers if you want to create + your listening sockets in some way other than + `~tornado.netutil.bind_sockets`. + + .. versionadded:: 3.1 + The ``max_buffer_size`` argument. + """ + def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None): + self.io_loop = io_loop + self.ssl_options = ssl_options + self._sockets = {} # fd -> socket object + self._pending_sockets = [] + self._started = False + self.max_buffer_size = max_buffer_size + + # Verify the SSL options. Otherwise we don't get errors until clients + # connect. This doesn't verify that the keys are legitimate, but + # the SSL module doesn't do that until there is a connected socket + # which seems like too much work + if self.ssl_options is not None and isinstance(self.ssl_options, dict): + # Only certfile is required: it can contain both keys + if 'certfile' not in self.ssl_options: + raise KeyError('missing key "certfile" in ssl_options') + + if not os.path.exists(self.ssl_options['certfile']): + raise ValueError('certfile "%s" does not exist' % + self.ssl_options['certfile']) + if ('keyfile' in self.ssl_options and + not os.path.exists(self.ssl_options['keyfile'])): + raise ValueError('keyfile "%s" does not exist' % + self.ssl_options['keyfile']) + + def listen(self, port, address=""): + """Starts accepting connections on the given port. + + This method may be called more than once to listen on multiple ports. + `listen` takes effect immediately; it is not necessary to call + `TCPServer.start` afterwards. It is, however, necessary to start + the `.IOLoop`. + """ + sockets = bind_sockets(port, address=address) + self.add_sockets(sockets) + + def add_sockets(self, sockets): + """Makes this server start accepting connections on the given sockets. + + The ``sockets`` parameter is a list of socket objects such as + those returned by `~tornado.netutil.bind_sockets`. + `add_sockets` is typically used in combination with that + method and `tornado.process.fork_processes` to provide greater + control over the initialization of a multi-process server. + """ + if self.io_loop is None: + self.io_loop = IOLoop.current() + + for sock in sockets: + self._sockets[sock.fileno()] = sock + add_accept_handler(sock, self._handle_connection, + io_loop=self.io_loop) + + def add_socket(self, socket): + """Singular version of `add_sockets`. Takes a single socket object.""" + self.add_sockets([socket]) + + def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128): + """Binds this server to the given port on the given address. + + To start the server, call `start`. If you want to run this server + in a single process, you can call `listen` as a shortcut to the + sequence of `bind` and `start` calls. + + Address may be either an IP address or hostname. If it's a hostname, + the server will listen on all IP addresses associated with the + name. Address may be an empty string or None to listen on all + available interfaces. Family may be set to either `socket.AF_INET` + or `socket.AF_INET6` to restrict to IPv4 or IPv6 addresses, otherwise + both will be used if available. + + The ``backlog`` argument has the same meaning as for + `socket.listen `. + + This method may be called multiple times prior to `start` to listen + on multiple ports or interfaces. + """ + sockets = bind_sockets(port, address=address, family=family, + backlog=backlog) + if self._started: + self.add_sockets(sockets) + else: + self._pending_sockets.extend(sockets) + + def start(self, num_processes=1): + """Starts this server in the `.IOLoop`. + + By default, we run the server in this process and do not fork any + additional child process. + + If num_processes is ``None`` or <= 0, we detect the number of cores + available on this machine and fork that number of child + processes. If num_processes is given and > 1, we fork that + specific number of sub-processes. + + Since we use processes and not threads, there is no shared memory + between any server code. + + Note that multiple processes are not compatible with the autoreload + module (or the ``debug=True`` option to `tornado.web.Application`). + When using multiple processes, no IOLoops can be created or + referenced until after the call to ``TCPServer.start(n)``. + """ + assert not self._started + self._started = True + if num_processes != 1: + process.fork_processes(num_processes) + sockets = self._pending_sockets + self._pending_sockets = [] + self.add_sockets(sockets) + + def stop(self): + """Stops listening for new connections. + + Requests currently in progress may still continue after the + server is stopped. + """ + for fd, sock in self._sockets.items(): + self.io_loop.remove_handler(fd) + sock.close() + + def handle_stream(self, stream, address): + """Override to handle a new `.IOStream` from an incoming connection.""" + raise NotImplementedError() + + def _handle_connection(self, connection, address): + if self.ssl_options is not None: + assert ssl, "Python 2.6+ and OpenSSL required for SSL" + try: + connection = ssl_wrap_socket(connection, + self.ssl_options, + server_side=True, + do_handshake_on_connect=False) + except ssl.SSLError as err: + if err.args[0] == ssl.SSL_ERROR_EOF: + return connection.close() + else: + raise + except socket.error as err: + # If the connection is closed immediately after it is created + # (as in a port scan), we can get one of several errors. + # wrap_socket makes an internal call to getpeername, + # which may return either EINVAL (Mac OS X) or ENOTCONN + # (Linux). If it returns ENOTCONN, this error is + # silently swallowed by the ssl module, so we need to + # catch another error later on (AttributeError in + # SSLIOStream._do_ssl_handshake). + # To test this behavior, try nmap with the -sT flag. + # https://github.com/facebook/tornado/pull/750 + if err.args[0] in (errno.ECONNABORTED, errno.EINVAL): + return connection.close() + else: + raise + try: + if self.ssl_options is not None: + stream = SSLIOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size) + else: + stream = IOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size) + self.handle_stream(stream, address) + except Exception: + app_log.error("Error in connection callback", exc_info=True) diff -Nru python-tornado-2.1.0/tornado/template.py python-tornado-3.1.1/tornado/template.py --- python-tornado-2.1.0/tornado/template.py 2011-09-21 04:21:52.000000000 +0000 +++ python-tornado-3.1.1/tornado/template.py 2013-09-01 18:41:35.000000000 +0000 @@ -79,9 +79,13 @@ to all templates by default. Typical applications do not create `Template` or `Loader` instances by -hand, but instead use the `render` and `render_string` methods of +hand, but instead use the `~.RequestHandler.render` and +`~.RequestHandler.render_string` methods of `tornado.web.RequestHandler`, which load templates automatically based -on the ``template_path`` `Application` setting. +on the ``template_path`` `.Application` setting. + +Variable names beginning with ``_tt_`` are reserved by the template +system and should not be used by application code. Syntax Reference ---------------- @@ -92,16 +96,24 @@ template directives use ``{% %}``. These tags may be escaped as ``{{!`` and ``{%!`` if you need to include a literal ``{{`` or ``{%`` in the output. +To comment out a section so that it is omitted from the output, surround it +with ``{# ... #}``. + ``{% apply *function* %}...{% end %}`` Applies a function to the output of all template code between ``apply`` and ``end``:: {% apply linkify %}{{name}} said: {{message}}{% end %} + Note that as an implementation detail apply blocks are implemented + as nested functions and thus may interact strangely with variables + set via ``{% set %}``, or the use of ``{% break %}`` or ``{% continue %}`` + within loops. + ``{% autoescape *function* %}`` Sets the autoescape mode for the current file. This does not affect other files, even those referenced by ``{% include %}``. Note that - autoescaping can also be configured globally, at the `Application` + autoescaping can also be configured globally, at the `.Application` or `Loader`.:: {% autoescape xhtml_escape %} @@ -131,8 +143,9 @@ tag will be ignored. For an example, see the ``{% block %}`` tag. ``{% for *var* in *expr* %}...{% end %}`` - Same as the python ``for`` statement. - + Same as the python ``for`` statement. ``{% break %}`` and + ``{% continue %}`` may be used inside the loop. + ``{% from *x* import *y* %}`` Same as the python ``import`` statement. @@ -162,34 +175,45 @@ ``{% set *x* = *y* %}`` Sets a local variable. -``{% try %}...{% except %}...{% finally %}...{% end %}`` +``{% try %}...{% except %}...{% finally %}...{% else %}...{% end %}`` Same as the python ``try`` statement. ``{% while *condition* %}... {% end %}`` - Same as the python ``while`` statement. + Same as the python ``while`` statement. ``{% break %}`` and + ``{% continue %}`` may be used inside the loop. """ -from __future__ import with_statement +from __future__ import absolute_import, division, print_function, with_statement -import cStringIO import datetime -import logging +import linecache import os.path import posixpath import re +import threading from tornado import escape -from tornado.util import bytes_type +from tornado.log import app_log +from tornado.util import bytes_type, ObjectDict, exec_in, unicode_type + +try: + from cStringIO import StringIO # py2 +except ImportError: + from io import StringIO # py3 _DEFAULT_AUTOESCAPE = "xhtml_escape" _UNSET = object() + class Template(object): """A compiled template. We compile into Python from the given template_string. You can generate the template from variables with generate(). """ + # note that the constructor's signature is not extracted with + # autodoc because _UNSET looks like garbage. When changing + # this signature update website/sphinx/template.rst too. def __init__(self, template_string, name="", loader=None, compress_whitespace=None, autoescape=_UNSET): self.name = name @@ -204,15 +228,21 @@ self.autoescape = _DEFAULT_AUTOESCAPE self.namespace = loader.namespace if loader else {} reader = _TemplateReader(name, escape.native_str(template_string)) - self.file = _File(_parse(reader, self)) + self.file = _File(self, _parse(reader, self)) self.code = self._generate_python(loader, compress_whitespace) + self.loader = loader try: - self.compiled = compile(escape.to_unicode(self.code), - "