diff -Nru swift-plugin-s3-1.0.0~git201200618/AUTHORS swift-plugin-s3-1.7/AUTHORS --- swift-plugin-s3-1.0.0~git201200618/AUTHORS 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/AUTHORS 2012-12-09 22:58:10.000000000 +0000 @@ -9,4 +9,5 @@ Michael Barton Rainer Toebbicke Scott Simpson +Tom Fifield Victor Rodionov diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/changelog swift-plugin-s3-1.7/debian/changelog --- swift-plugin-s3-1.0.0~git201200618/debian/changelog 2012-09-28 17:33:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/changelog 2014-02-05 05:08:30.000000000 +0000 @@ -1,11 +1,27 @@ -swift-plugin-s3 (1.0.0~git201200618-0ubuntu1~cloud0) precise-folsom; urgency=low +swift-plugin-s3 (1.7-3~cloud0) precise-icehouse; urgency=low - * New snapshot for the Ubuntu Cloud Archive. + * New update for the Ubuntu Cloud Archive. - -- Chuck Short Fri, 28 Sep 2012 12:32:40 -0500 + -- Openstack Ubuntu Testing Bot Wed, 05 Feb 2014 00:08:30 -0500 -swift-plugin-s3 (1.0.0~git201200618-0ubuntu1) quantal; urgency=low +swift-plugin-s3 (1.7-3) unstable; urgency=medium - * Initial release. + * Ran wrap-and-sort. + * Added upstream patches cherry-pick from github (Closes: #737639): + - fix_signature_bug_to_use_RAW_PATH_INFO.patch + - Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch + * Standards-Version: is now 3.9.5. - -- Chuck Short Mon, 18 Jun 2012 10:30:29 -0400 + -- Thomas Goirand Tue, 04 Feb 2014 15:10:16 +0000 + +swift-plugin-s3 (1.7-2) unstable; urgency=low + + * Uploading to unstable. + + -- Thomas Goirand Fri, 17 May 2013 08:09:11 +0000 + +swift-plugin-s3 (1.7-1) experimental; urgency=low + + * Initial release (Closes: #693137). + + -- Thomas Goirand Tue, 13 Nov 2012 14:05:57 +0000 diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/compat swift-plugin-s3-1.7/debian/compat --- swift-plugin-s3-1.0.0~git201200618/debian/compat 2012-06-18 14:30:31.000000000 +0000 +++ swift-plugin-s3-1.7/debian/compat 2014-02-04 15:11:20.000000000 +0000 @@ -1 +1 @@ -8 +9 diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/control swift-plugin-s3-1.7/debian/control --- swift-plugin-s3-1.0.0~git201200618/debian/control 2012-06-18 14:54:29.000000000 +0000 +++ swift-plugin-s3-1.7/debian/control 2014-02-04 15:11:20.000000000 +0000 @@ -1,15 +1,49 @@ Source: swift-plugin-s3 Section: python Priority: extra -Maintainer: Chuck Short -Build-Depends: debhelper (>= 8.0.0), python-setuptools, python-all (>= 2.6.6-3~), - python-webob, python-simplejson -Standards-Version: 3.9.3 +Maintainer: PKG OpenStack +Uploaders: Loic Dachary (OuoU) , + Julien Danjou , + Thomas Goirand , + Ghe Rivero , + Mehdi Abaakouk +Build-Depends: debhelper (>= 9), python-all (>= 2.6.6-3~) +Build-Depends-Indep: openstack-pkg-tools, + python-setuptools, + python-simplejson, + python-webob +Standards-Version: 3.9.5 +Vcs-Browser: http://anonscm.debian.org/gitweb/?p=openstack/swift-plugin-s3.git;a=summary +Vcs-Git: git://anonscm.debian.org/openstack/swift-plugin-s3.git +Homepage: https://github.com/fujita/swift3/ Package: swift-plugin-s3 Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, python-webob, python-simplejson, - python-swift -Description: swift3 middleware for swift. - Swift3 Middleware for OpenStack Swift, allowing access to OpenStack - swift via the Amazon S3 API. +Pre-Depends: dpkg (>= 1.15.6~) +Depends: python-simplejson, + python-swift, + python-webob, + ${misc:Depends}, + ${python:Depends} +Description: swift3 (S3 compatibility) middleware plugin for swift + OpenStack Object Storage (code-named Swift) creates redundant, scalable object + storage using clusters of standardized servers to store petabytes of + accessible data. It is not a file system or real-time data storage system, but + rather a long-term storage system for a more permanent type of static data + that can be retrieved, leveraged, and then updated if necessary. Primary + examples of data that best fit this type of storage model are virtual machine + images, photo storage, email storage and backup archiving. Having no central + "brain" or master point of control provides greater scalability, redundancy + and permanence. + . + Objects are written to multiple hardware devices in the data center, with + the OpenStack software responsible for ensuring data replication and + integrity across the cluster. Storage clusters can scale horizontally by + adding new nodes. Should a node fail, OpenStack works to replicate its + content from other active nodes. Because OpenStack uses software logic to + ensure data replication and distribution across different devices, + inexpensive commodity hard drives and servers can be used in lieu of more + expensive equipment. + . + This package provides the Swift3 Middleware plugin for OpenStack Swift, + allowing access to OpenStack swift via the Amazon S3 API. diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/copyright swift-plugin-s3-1.7/debian/copyright --- swift-plugin-s3-1.0.0~git201200618/debian/copyright 2012-06-18 14:45:59.000000000 +0000 +++ swift-plugin-s3-1.7/debian/copyright 2014-02-04 15:11:20.000000000 +0000 @@ -1,9 +1,16 @@ -Format: http://dep.debian.net/deps/dep5 +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: swift3 +Upstream-Contact: FUJITA Tomonori Source: https://github.com/fujita/swift3 +Files: debian/* +Copyright: (c) 2012, Thomas Goirand +License: Apache-2 + Files: * -Copyright: 2012 Openstack LLC. +Copyright: 2012 Openstack LLC. (see AUTHORS file) +License: Apache-2 + License: Apache-2 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,23 +25,4 @@ limitations under the License. . On Debian-based systems the full text of the Apache version 2.0 license - can be found in `/usr/share/common-licenses/Apache-2.0'. - -Files: debian/* -Copyright: 2012 Canonical Ltd -License: GPL-2+ - This package is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - . - This package is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - . - You should have received a copy of the GNU General Public License - along with this program. If not, see - . - On Debian systems, the complete text of the GNU General - Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". + can be found in "/usr/share/common-licenses/Apache-2.0". diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/docs swift-plugin-s3-1.7/debian/docs --- swift-plugin-s3-1.0.0~git201200618/debian/docs 2012-06-18 14:41:04.000000000 +0000 +++ swift-plugin-s3-1.7/debian/docs 2014-02-04 15:11:20.000000000 +0000 @@ -1 +1,2 @@ README.md +AUTHORS diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/gbp.conf swift-plugin-s3-1.7/debian/gbp.conf --- swift-plugin-s3-1.0.0~git201200618/debian/gbp.conf 1970-01-01 00:00:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/gbp.conf 2014-02-04 15:11:20.000000000 +0000 @@ -0,0 +1,8 @@ +[DEFAULT] +upstream-branch = master +debian-branch = debian/unstable +upstream-tag = v%(version)s +compression = xz + +[git-buildpackage] +export-dir = ../build-area/ diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/patches/fix_signature_bug_to_use_RAW_PATH_INFO.patch swift-plugin-s3-1.7/debian/patches/fix_signature_bug_to_use_RAW_PATH_INFO.patch --- swift-plugin-s3-1.0.0~git201200618/debian/patches/fix_signature_bug_to_use_RAW_PATH_INFO.patch 1970-01-01 00:00:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/patches/fix_signature_bug_to_use_RAW_PATH_INFO.patch 2014-02-04 15:11:20.000000000 +0000 @@ -0,0 +1,46 @@ +From b8b5998e0a9fad4e63dc8db2085e8bfbc387dec4 Mon Sep 17 00:00:00 2001 +From: Kota Tsuyuzaki +Date: Tue, 26 Feb 2013 20:57:32 -0800 +Subject: [PATCH] fix signature bug to use RAW_PATH_INFO + +This fixes signature creation to use RAW_PATH_INFO. + +Swift3 could not create correct signature in case of +using escaped character(e.g. %2F, %2D) in PATH_INFO, +because env['PATH_INFO'] was decoded(unescaped) by +eventlet.wsgi before a request arrived at swift3. +It caused signature mismatch and authentication failure. + +This change enables swift3 to create signature from +RAW_PATH_INFO and fixes that bug. + +Note: This patch works well only in later version than + eventlet 0.9.17, because older version does not + have RAW_PATH_INFO variable. + When using older version, swift3 works in the same + way as ever(use req.path of swob). + +Signed-off-by: Kota Tsuyuzaki +--- + swift3/middleware.py | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/swift3/middleware.py b/swift3/middleware.py +index 87159b3..b009b09 100644 +--- a/swift3/middleware.py ++++ b/swift3/middleware.py +@@ -266,7 +266,10 @@ def canonical_string(req): + for k in sorted(key.lower() for key in amz_headers): + buf += "%s:%s\n" % (k, amz_headers[k]) + +- path = req.path ++ # RAW_PATH_INFO is enabled in later version than eventlet 0.9.17. ++ # When using older version, swift3 uses req.path of swob instead ++ # of it. ++ path = req.environ.get('RAW_PATH_INFO', req.path) + if req.query_string: + path += '?' + req.query_string + if '?' in path: +-- +1.8.5.1 + diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/patches/series swift-plugin-s3-1.7/debian/patches/series --- swift-plugin-s3-1.0.0~git201200618/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/patches/series 2014-02-04 15:11:20.000000000 +0000 @@ -0,0 +1,2 @@ +fix_signature_bug_to_use_RAW_PATH_INFO.patch +Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/patches/Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch swift-plugin-s3-1.7/debian/patches/Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch --- swift-plugin-s3-1.0.0~git201200618/debian/patches/Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch 1970-01-01 00:00:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/patches/Updated_HEAD_and_ACL_calls_to_objects_to_work_correctly.patch 2014-02-04 15:11:20.000000000 +0000 @@ -0,0 +1,59 @@ +From 47148674264dd4b07ff396f3481bbba203937491 Mon Sep 17 00:00:00 2001 +From: Chuck Thier +Date: Thu, 6 Dec 2012 11:06:08 -0600 +Subject: [PATCH] Updated HEAD and ACL calls to objects to work correctly + +--- + swift3/middleware.py | 20 +++++++++----------- + 1 file changed, 9 insertions(+), 11 deletions(-) + +diff --git a/swift3/middleware.py b/swift3/middleware.py +index bf52679..192a882 100644 +--- a/swift3/middleware.py ++++ b/swift3/middleware.py +@@ -41,7 +41,8 @@ + An example client using the python boto library might look like the + following for an SAIO setup:: + +- connection = boto.s3.Connection( ++ from boto.s3.connection import S3Connection ++ connection = S3Connection( + aws_access_key_id='test:tester', + aws_secret_access_key='testing', + port=8080, +@@ -603,25 +604,22 @@ def __init__(self, env, app, account_name, token, container_name, + object_name) + + def GETorHEAD(self, env, start_response): +- if env['REQUEST_METHOD'] == 'HEAD': +- head = True +- env['REQUEST_METHOD'] = 'GET' ++ if 'QUERY_STRING' in env: ++ args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) + else: +- head = False ++ args = {} ++ if 'acl' in args: ++ # ACL requests need to make a HEAD call rather than GET ++ env['REQUEST_METHOD'] = 'HEAD' + + app_iter = self._app_call(env) +- +- if head: ++ if env['REQUEST_METHOD'] == 'HEAD': + app_iter = None + + status = self._get_status_int() + headers = dict(self._response_headers) + + if is_success(status): +- if 'QUERY_STRING' in env: +- args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) +- else: +- args = {} + if 'acl' in args: + return get_acl(self.account_name, headers) + +-- +1.8.5.1 + diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/README.source swift-plugin-s3-1.7/debian/README.source --- swift-plugin-s3-1.0.0~git201200618/debian/README.source 2012-06-18 14:34:42.000000000 +0000 +++ swift-plugin-s3-1.7/debian/README.source 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -swift-plugin-s3 for Debian --------------------------- - -This package source was generated by doing the following: - -git clone git://github.com/fujita/swift3.git -cd swift3.git -git archive --format=tar --prefix=swift-plugin-s3-1.0.0~git201200618/ HEAD | gzip > swift-plugin-s3-1.0.0~git201200618.tar.gz diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/rules swift-plugin-s3-1.7/debian/rules --- swift-plugin-s3-1.0.0~git201200618/debian/rules 2012-06-18 14:44:04.000000000 +0000 +++ swift-plugin-s3-1.7/debian/rules 2014-02-04 15:11:20.000000000 +0000 @@ -1,8 +1,13 @@ #!/usr/bin/make -f -# Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 +UPSTREAM_GIT := git://github.com/fujita/swift3.git + +include /usr/share/openstack-pkg-tools/pkgos.make + +GIT_TAG := v$(shell echo '$(VERSION)' | sed -e 's/~/_/') + %: dh $@ --with python2 diff -Nru swift-plugin-s3-1.0.0~git201200618/debian/watch swift-plugin-s3-1.7/debian/watch --- swift-plugin-s3-1.0.0~git201200618/debian/watch 1970-01-01 00:00:00.000000000 +0000 +++ swift-plugin-s3-1.7/debian/watch 2014-02-04 15:11:20.000000000 +0000 @@ -0,0 +1,2 @@ +version=3 +https://github.com/fujita/swift3/tags .*/v(\d[\d\.]+)\.tar\.gz diff -Nru swift-plugin-s3-1.0.0~git201200618/doc/source/conf.py swift-plugin-s3-1.7/doc/source/conf.py --- swift-plugin-s3-1.0.0~git201200618/doc/source/conf.py 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/doc/source/conf.py 2012-12-09 22:58:10.000000000 +0000 @@ -39,7 +39,7 @@ master_doc = 'index' # General information about the project. -project = u'Swift Static Web' +project = u'Swift S3 Compatibility Middleware' copyright = u'2012 OpenStack, LLC.' # The version info for the project you're documenting, acts as replacement for diff -Nru swift-plugin-s3-1.0.0~git201200618/doc/source/index.rst swift-plugin-s3-1.7/doc/source/index.rst --- swift-plugin-s3-1.0.0~git201200618/doc/source/index.rst 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/doc/source/index.rst 2012-12-09 22:58:10.000000000 +0000 @@ -1,5 +1,5 @@ -Swift Static Web -**************** +Swift S3 Compatibility Middleware +********************************* Copyright 2012 OpenStack, LLC. diff -Nru swift-plugin-s3-1.0.0~git201200618/.gitignore swift-plugin-s3-1.7/.gitignore --- swift-plugin-s3-1.0.0~git201200618/.gitignore 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/.gitignore 2012-12-09 22:58:10.000000000 +0000 @@ -1,3 +1,4 @@ *.egg-info *.py[co] .DS_Store +.idea diff -Nru swift-plugin-s3-1.0.0~git201200618/swift3/__init__.py swift-plugin-s3-1.7/swift3/__init__.py --- swift-plugin-s3-1.0.0~git201200618/swift3/__init__.py 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/swift3/__init__.py 2012-12-09 22:58:10.000000000 +0000 @@ -19,6 +19,6 @@ __all__ = ['version_info', 'version'] #: Version information ``(major, minor, revision)``. -version_info = (1, 0, 0) +version_info = (1, 7, 0) #: Version string ``'major.minor.revision'``. version = '.'.join(map(str, version_info)) diff -Nru swift-plugin-s3-1.0.0~git201200618/swift3/middleware.py swift-plugin-s3-1.7/swift3/middleware.py --- swift-plugin-s3-1.0.0~git201200618/swift3/middleware.py 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/swift3/middleware.py 2012-12-09 22:58:10.000000000 +0000 @@ -54,17 +54,21 @@ import base64 from xml.sax.saxutils import escape as xml_escape import urlparse +from xml.dom.minidom import parseString -from webob import Request, Response from simplejson import loads import email.utils import datetime +import re from swift.common.utils import split_path +from swift.common.utils import get_logger from swift.common.wsgi import WSGIContext +from swift.common.swob import Request, Response from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ HTTP_NO_CONTENT, HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, \ - HTTP_NOT_FOUND, HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, is_success + HTTP_NOT_FOUND, HTTP_CONFLICT, HTTP_UNPROCESSABLE_ENTITY, is_success, \ + HTTP_NOT_IMPLEMENTED, HTTP_LENGTH_REQUIRED, HTTP_SERVICE_UNAVAILABLE MAX_BUCKET_LISTING = 1000 @@ -79,55 +83,163 @@ """ error_table = { 'AccessDenied': - (HTTP_FORBIDDEN, 'Access denied'), + (HTTP_FORBIDDEN, 'Access denied'), 'BucketAlreadyExists': - (HTTP_CONFLICT, 'The requested bucket name is not available'), + (HTTP_CONFLICT, 'The requested bucket name is not available'), 'BucketNotEmpty': - (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), + (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), 'InvalidArgument': - (HTTP_BAD_REQUEST, 'Invalid Argument'), + (HTTP_BAD_REQUEST, 'Invalid Argument'), 'InvalidBucketName': - (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), + (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), 'InvalidURI': - (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), + (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), 'InvalidDigest': - (HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'), + (HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'), + 'BadDigest': + (HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'), 'NoSuchBucket': - (HTTP_NOT_FOUND, 'The specified bucket does not exist'), + (HTTP_NOT_FOUND, 'The specified bucket does not exist'), 'SignatureDoesNotMatch': - (HTTP_FORBIDDEN, 'The calculated request signature does not '\ + (HTTP_FORBIDDEN, 'The calculated request signature does not ' 'match your provided one'), 'RequestTimeTooSkewed': - (HTTP_FORBIDDEN, 'The difference between the request time and '\ - ' the current time is too large'), + (HTTP_FORBIDDEN, 'The difference between the request time and the' + ' current time is too large'), 'NoSuchKey': - (HTTP_NOT_FOUND, 'The resource you requested does not exist')} + (HTTP_NOT_FOUND, 'The resource you requested does not exist'), + 'Unsupported': + (HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet' + ' implemented'), + 'MissingContentLength': + (HTTP_LENGTH_REQUIRED, 'Length Required'), + 'ServiceUnavailable': + (HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate')} resp = Response(content_type='text/xml') resp.status = error_table[code][0] - resp.body = error_table[code][1] resp.body = '\r\n\r\n ' \ '%s\r\n %s\r\n\r\n' \ - % (code, error_table[code][1]) + % (code, error_table[code][1]) return resp -def get_acl(account_name): - body = ('' - '' - '%s' - '' - '' - '' - '' - '%s' - '' - 'FULL_CONTROL' - '' - '' - '' % - (account_name, account_name)) +def get_acl(account_name, headers): + """ + Attempts to construct an S3 ACL based on what is found in the swift headers + """ + + acl = 'private' # default to private + + if 'x-container-read' in headers: + if headers['x-container-read'] == ".r:*" or\ + ".r:*," in headers['x-container-read'] or \ + ",*," in headers['x-container-read']: + acl = 'public-read' + if 'x-container-write' in headers: + if headers['x-container-write'] == ".r:*" or\ + ".r:*," in headers['x-container-write'] or \ + ",*," in headers['x-container-write']: + if acl == 'public-read': + acl = 'public-read-write' + else: + acl = 'public-write' + + if acl == 'private': + body = ('' + '' + '%s' + '%s' + '' + '' + '' + '' + '%s' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' % + (account_name, account_name, account_name, account_name)) + elif acl == 'public-read': + body = ('' + '' + '%s' + '%s' + '' + '' + '' + '' + '%s' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' + 'http://acs.amazonaws.com/groups/global/AllUsers' + '' + 'READ' + '' + '' + '' % + (account_name, account_name, account_name, account_name)) + elif acl == 'public-read-write': + body = ('' + '' + '%s' + '%s' + '' + '' + '' + '' + '%s' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' + 'http://acs.amazonaws.com/groups/global/AllUsers' + '' + 'READ' + '' + '' + '' + '' + '' + 'http://acs.amazonaws.com/groups/global/AllUsers' + '' + 'WRITE' + '' + '' + '' % + (account_name, account_name, account_name, account_name)) + else: + body = ('' + '' + '%s' + '%s' + '' + '' + '' + '' + '%s' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' % + (account_name, account_name, account_name, account_name)) return Response(body=body, content_type="text/plain") @@ -138,7 +250,7 @@ amz_headers = {} buf = "%s\n%s\n%s\n" % (req.method, req.headers.get('Content-MD5', ''), - req.headers.get('Content-Type') or '') + req.headers.get('Content-Type') or '') for amz_header in sorted((key.lower() for key in req.headers if key.lower().startswith('x-amz-'))): @@ -152,16 +264,91 @@ for k in sorted(key.lower() for key in amz_headers): buf += "%s:%s\n" % (k, amz_headers[k]) - path = req.path_qs + path = req.path + if req.query_string: + path += '?' + req.query_string if '?' in path: path, args = path.split('?', 1) for key in urlparse.parse_qs(args, keep_blank_values=True): if key in ('acl', 'logging', 'torrent', 'location', - 'requestPayment'): + 'requestPayment', 'versioning'): return "%s%s?%s" % (buf, path, key) return buf + path +def swift_acl_translate(acl, group='', user='', xml=False): + """ + Takes an S3 style ACL and returns a list of header/value pairs that + implement that ACL in Swift, or "Unsupported" if there isn't a way to do + that yet. + """ + swift_acl = {} + swift_acl['public-read'] = [['HTTP_X_CONTAINER_READ', '.r:*,.rlistings']] + # Swift does not support public write: + # https://answers.launchpad.net/swift/+question/169541 + swift_acl['public-read-write'] = [['HTTP_X_CONTAINER_WRITE', '.r:*'], + ['HTTP_X_CONTAINER_READ', + '.r:*,.rlistings']] + + #TODO: if there's a way to get group and user, this should work for + # private: + #swift_acl['private'] = [['HTTP_X_CONTAINER_WRITE', group + ':' + user], \ + # ['HTTP_X_CONTAINER_READ', group + ':' + user]] + swift_acl['private'] = [['HTTP_X_CONTAINER_WRITE', '.'], + ['HTTP_X_CONTAINER_READ', '.']] + if xml: + # We are working with XML and need to parse it + dom = parseString(acl) + acl = 'unknown' + for grant in dom.getElementsByTagName('Grant'): + permission = grant.getElementsByTagName('Permission')[0]\ + .firstChild.data + grantee = grant.getElementsByTagName('Grantee')[0]\ + .getAttributeNode('xsi:type').nodeValue + if permission == "FULL_CONTROL" and grantee == 'CanonicalUser' and\ + acl != 'public-read' and acl != 'public-read-write': + acl = 'private' + elif permission == "READ" and grantee == 'Group' and\ + acl != 'public-read-write': + acl = 'public-read' + elif permission == "WRITE" and grantee == 'Group': + acl = 'public-read-write' + else: + acl = 'unsupported' + + if acl == 'authenticated-read': + return "Unsupported" + elif acl not in swift_acl: + return "InvalidArgument" + + return swift_acl[acl] + + +def validate_bucket_name(name): + """ + Validates the name of the bucket against S3 criteria, + http://docs.amazonwebservices.com/AmazonS3/latest/BucketRestrictions.html + True if valid, False otherwise + """ + + if '_' in name or len(name) < 3 or len(name) > 63 or not name[-1].isalnum(): + # Bucket names should not contain underscores (_) + # Bucket names must end with a lowercase letter or number + # Bucket names should be between 3 and 63 characters long + return False + elif '.-' in name or '-.' in name or '..' in name or not name[0].isalnum(): + # Bucket names cannot contain dashes next to periods + # Bucket names cannot contain two adjacent periods + # Bucket names Must start with a lowercase letter or a number + return False + elif re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}" + "([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", name): + # Bucket names cannot be formatted as an IP Address + return False + else: + return True + + class ServiceController(WSGIContext): """ Handles account level requests. @@ -189,15 +376,15 @@ # we don't keep the creation time of a backet (s3cmd doesn't # work without that) so we use something bogus. body = '' \ - '' \ - '%s' \ - '' \ - % ("".join(['%s' \ - '2009-02-03T16:45:09.000Z' % - xml_escape(i['name']) for i in containers])) + '' \ + '%s' \ + '' \ + % ("".join(['%s' + '2009-02-03T16:45:09.000Z' + % xml_escape(i['name']) for i in containers])) resp = Response(status=HTTP_OK, content_type='application/xml', - body=body) + body=body) return resp @@ -206,12 +393,14 @@ Handles bucket request. """ def __init__(self, env, app, account_name, token, container_name, - **kwargs): + **kwargs): WSGIContext.__init__(self, app) self.container_name = unquote(container_name) self.account_name = unquote(account_name) env['HTTP_X_AUTH_TOKEN'] = token env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name) + conf = kwargs.get('conf', {}) + self.location = conf.get('location', 'US') def GET(self, env, start_response): """ @@ -221,9 +410,17 @@ args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) else: args = {} + + if 'max-keys' in args: + if args.get('max-keys').isdigit() is False: + return get_err_response('InvalidArgument') + max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)), - MAX_BUCKET_LISTING) - env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1) + MAX_BUCKET_LISTING) + + if 'acl' not in args: + #acl request sent with format=json etc confuses swift + env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1) if 'marker' in args: env['QUERY_STRING'] += '&marker=%s' % quote(args['marker']) if 'prefix' in args: @@ -232,6 +429,16 @@ env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter']) body_iter = self._app_call(env) status = self._get_status_int() + headers = dict(self._response_headers) + + if 'acl' in args: + return get_acl(self.account_name, headers) + + if 'versioning' in args: + # Just report there is no versioning configured here. + body = ('') + return Response(body=body, content_type="text/plain") if status != HTTP_OK: if status == HTTP_UNAUTHORIZED: @@ -241,35 +448,52 @@ else: return get_err_response('InvalidURI') - if 'acl' in args: - return get_acl(self.account_name) + if 'location' in args: + body = ('' + '%s' % self.location) + return Response(body=body, content_type='application/xml') + + if 'logging' in args: + # logging disabled + body = ('' + '') + return Response(body=body, content_type='application/xml') objects = loads(''.join(list(body_iter))) body = ('' - '' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '%s' - '' % - ( + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '' % + ( xml_escape(args.get('prefix', '')), xml_escape(args.get('marker', '')), xml_escape(args.get('delimiter', '')), - 'true' if len(objects) == (max_keys + 1) else 'false', + 'true' if max_keys > 0 and len(objects) == (max_keys + 1) else + 'false', max_keys, xml_escape(self.container_name), - "".join(['%s%sZ%s%sSTA'\ - 'NDARD' % - (xml_escape(i['name']), i['last_modified'], i['hash'], - i['bytes']) - for i in objects[:max_keys] if 'subdir' not in i]), + "".join(['%s%sZ%s%sSTA' + 'NDARD%s' + '%s' % + (xml_escape(unquote(i['name'])), i['last_modified'], + i['hash'], + i['bytes'], self.account_name, self.account_name) + for i in objects[:max_keys] if 'subdir' not in i]), "".join(['%s' % xml_escape(i['subdir']) for i in objects[:max_keys] if 'subdir' in i]))) @@ -279,10 +503,51 @@ """ Handle PUT Bucket request """ + if 'HTTP_X_AMZ_ACL' in env: + amz_acl = env['HTTP_X_AMZ_ACL'] + # Translate the Amazon ACL to something that can be + # implemented in Swift, 501 otherwise. Swift uses POST + # for ACLs, whereas S3 uses PUT. + del env['HTTP_X_AMZ_ACL'] + if 'QUERY_STRING' in env: + del env['QUERY_STRING'] + + translated_acl = swift_acl_translate(amz_acl) + if translated_acl == 'Unsupported': + return get_err_response('Unsupported') + elif translated_acl == 'InvalidArgument': + return get_err_response('InvalidArgument') + + for header, acl in translated_acl: + env[header] = acl + + if 'CONTENT_LENGTH' in env: + content_length = env['CONTENT_LENGTH'] + try: + content_length = int(content_length) + except (ValueError, TypeError): + return get_err_response('InvalidArgument') + if content_length < 0: + return get_err_response('InvalidArgument') + + if 'QUERY_STRING' in env: + args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) + if 'acl' in args: + # We very likely have an XML-based ACL request. + body = env['wsgi.input'].readline().decode() + translated_acl = swift_acl_translate(body, xml=True) + if translated_acl == 'Unsupported': + return get_err_response('Unsupported') + elif translated_acl == 'InvalidArgument': + return get_err_response('InvalidArgument') + for header, acl in translated_acl: + env[header] = acl + env['REQUEST_METHOD'] = 'POST' + body_iter = self._app_call(env) status = self._get_status_int() - if status != HTTP_CREATED: + if status != HTTP_CREATED and status != HTTP_NO_CONTENT: if status == HTTP_UNAUTHORIZED: return get_err_response('AccessDenied') elif status == HTTP_ACCEPTED: @@ -291,7 +556,7 @@ return get_err_response('InvalidURI') resp = Response() - resp.headers.add('Location', self.container_name) + resp.headers['Location'] = self.container_name resp.status = HTTP_OK return resp @@ -316,13 +581,20 @@ resp.status = HTTP_NO_CONTENT return resp + def POST(self, env, start_response): + """ + Handle POST Bucket request + """ + + return get_err_response('Unsupported') + class ObjectController(WSGIContext): """ Handles requests on objects """ def __init__(self, env, app, account_name, token, container_name, - object_name, **kwargs): + object_name, **kwargs): WSGIContext.__init__(self, app) self.account_name = unquote(account_name) self.container_name = unquote(container_name) @@ -331,7 +603,17 @@ object_name) def GETorHEAD(self, env, start_response): + if env['REQUEST_METHOD'] == 'HEAD': + head = True + env['REQUEST_METHOD'] = 'GET' + else: + head = False + app_iter = self._app_call(env) + + if head: + app_iter = None + status = self._get_status_int() headers = dict(self._response_headers) @@ -341,7 +623,7 @@ else: args = {} if 'acl' in args: - return get_acl(self.account_name) + return get_acl(self.account_name, headers) new_hdrs = {} for key, val in headers.iteritems(): @@ -437,6 +719,8 @@ """Swift3 S3 compatibility midleware""" def __init__(self, app, conf, *args, **kwargs): self.app = app + self.conf = conf + self.logger = get_logger(self.conf, log_route='swift3') def get_controller(self, path): container, obj = split_path(path, 0, 2, True) @@ -449,17 +733,26 @@ return ServiceController, d def __call__(self, env, start_response): + try: + return self.handle_request(env, start_response) + except Exception, e: + self.logger.exception(e) + return get_err_response('ServiceUnavailable')(env, start_response) + + def handle_request(self, env, start_response): req = Request(env) + self.logger.debug('Calling Swift3 Middleware') + self.logger.debug(req.__dict__) - if 'AWSAccessKeyId' in req.GET: + if 'AWSAccessKeyId' in req.params: try: - req.headers['Date'] = req.GET['Expires'] + req.headers['Date'] = req.params['Expires'] req.headers['Authorization'] = \ - 'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET + 'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.params except KeyError: return get_err_response('InvalidArgument')(env, start_response) - if not 'Authorization' in req.headers: + if 'Authorization' not in req.headers: return self.app(env, start_response) try: @@ -482,7 +775,7 @@ if 'Date' in req.headers: date = email.utils.parsedate(req.headers['Date']) - if date == None: + if date is None: return get_err_response('AccessDenied')(env, start_response) d1 = datetime.datetime(*date[0:6]) @@ -494,11 +787,13 @@ delta = datetime.timedelta(seconds=60 * 10) if d1 - d2 > delta or d2 - d1 > delta: - return get_err_response('RequestTimeTooSkewed')(env, start_response) + return get_err_response('RequestTimeTooSkewed')(env, + start_response) token = base64.urlsafe_b64encode(canonical_string(req)) - controller = controller(env, self.app, account, token, **path_parts) + controller = controller(env, self.app, account, token, conf=self.conf, + **path_parts) if hasattr(controller, req.method): res = getattr(controller, req.method)(env, start_response) diff -Nru swift-plugin-s3-1.0.0~git201200618/swift3/test/unit/test_swift3.py swift-plugin-s3-1.7/swift3/test/unit/test_swift3.py --- swift-plugin-s3-1.0.0~git201200618/swift3/test/unit/test_swift3.py 2012-06-17 07:06:54.000000000 +0000 +++ swift-plugin-s3-1.7/swift3/test/unit/test_swift3.py 2012-12-09 22:58:10.000000000 +0000 @@ -18,12 +18,13 @@ import cgi import hashlib -from webob import Request, Response -from webob.exc import HTTPUnauthorized, HTTPCreated, HTTPNoContent,\ - HTTPAccepted, HTTPBadRequest, HTTPNotFound, HTTPConflict import xml.dom.minidom import simplejson +from swift.common.swob import Request, Response, HTTPUnauthorized, \ + HTTPCreated,HTTPNoContent, HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ + HTTPConflict + from swift3 import middleware as swift3 @@ -129,41 +130,42 @@ 'last-modified': '2011-01-05T02:19:14.275290'} def __call__(self, env, start_response): + req = Request(env) if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD': if self.status == 200: if 'HTTP_RANGE' in env: - resp = Response(body=self.object_body, + resp = Response(request=req, body=self.object_body, conditional_response=True) return resp(env, start_response) - start_response(Response().status, + start_response(Response(request=req).status, self.response_headers.items()) if env['REQUEST_METHOD'] == 'GET': return self.object_body elif self.status == 401: - start_response(HTTPUnauthorized().status, []) + start_response(HTTPUnauthorized(request=req).status, []) elif self.status == 404: - start_response(HTTPNotFound().status, []) + start_response(HTTPNotFound(request=req).status, []) else: - start_response(HTTPBadRequest().status, []) + start_response(HTTPBadRequest(request=req).status, []) elif env['REQUEST_METHOD'] == 'PUT': if self.status == 201: - start_response(HTTPCreated().status, + start_response(HTTPCreated(request=req).status, [('etag', self.response_headers['etag'])]) elif self.status == 401: - start_response(HTTPUnauthorized().status, []) + start_response(HTTPUnauthorized(request=req).status, []) elif self.status == 404: - start_response(HTTPNotFound().status, []) + start_response(HTTPNotFound(request=req).status, []) else: - start_response(HTTPBadRequest().status, []) + start_response(HTTPBadRequest(request=req).status, []) elif env['REQUEST_METHOD'] == 'DELETE': if self.status == 204: - start_response(HTTPNoContent().status, []) + start_response(HTTPNoContent(request=req).status, []) elif self.status == 401: - start_response(HTTPUnauthorized().status, []) + start_response(HTTPUnauthorized(request=req).status, []) elif self.status == 404: - start_response(HTTPNotFound().status, []) + start_response(HTTPNotFound(request=req).status, []) else: - start_response(HTTPBadRequest().status, []) + start_response(HTTPBadRequest(request=req).status, []) return [] @@ -187,7 +189,7 @@ dom = xml.dom.minidom.parseString("".join(resp)) self.assertEquals(dom.firstChild.nodeName, 'Error') code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue - self.assertEquals(code, 'InvalidArgument') + self.assertEquals(code, 'AccessDenied') def test_bad_method(self): req = Request.blank('/', @@ -199,11 +201,11 @@ code = dom.getElementsByTagName('Code')[0].childNodes[0].nodeValue self.assertEquals(code, 'InvalidURI') - def _test_method_error(self, cl, method, path, status): + def _test_method_error(self, cl, method, path, status, headers={}): local_app = swift3.filter_factory({})(cl(status)) - req = Request.blank(path, - environ={'REQUEST_METHOD': method}, - headers={'Authorization': 'AWS test:tester:hmac'}) + headers.update({'Authorization': 'AWS test:tester:hmac'}) + req = Request.blank(path, environ={'REQUEST_METHOD': method}, + headers=headers) resp = local_app(req.environ, start_response) dom = xml.dom.minidom.parseString("".join(resp)) self.assertEquals(dom.firstChild.nodeName, 'Error') @@ -262,7 +264,6 @@ self.assertEquals(name, bucket_name) objects = [n for n in dom.getElementsByTagName('Contents')] - listing = [n for n in objects[0].childNodes if n.nodeName != '#text'] names = [] for o in objects: @@ -357,6 +358,12 @@ self.assertEquals(args['prefix'], 'c') def test_bucket_PUT_error(self): + code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 201, + headers={'Content-Length': 'a'}) + self.assertEqual(code, 'InvalidArgument') + code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 201, + headers={'Content-Length': '-1'}) + self.assertEqual(code, 'InvalidArgument') code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 401) self.assertEquals(code, 'AccessDenied') code = self._test_method_error(FakeAppBucket, 'PUT', '/bucket', 202) @@ -407,6 +414,16 @@ resp = local_app(req.environ, local_app.app.do_start_response) self._check_acl('test:tester', resp) + def test_bucket_versioning_GET(self): + local_app = swift3.filter_factory({})(FakeAppBucket()) + bucket_name = 'junk' + req = Request.blank('/%s?versioning' % bucket_name, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + resp = local_app(req.environ, local_app.app.do_start_response) + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.firstChild.nodeName, 'VersioningConfiguration') + def _test_object_GETorHEAD(self, method): local_app = swift3.filter_factory({})(FakeAppObject()) req = Request.blank('/bucket/object', @@ -415,9 +432,10 @@ resp = local_app(req.environ, local_app.app.do_start_response) self.assertEquals(local_app.app.response_args[0].split()[0], '200') - headers = dict(local_app.app.response_args[1]) + headers = dict((k.lower(), v) for k, v in + local_app.app.response_args[1]) for key, val in local_app.app.response_headers.iteritems(): - if key in ('Content-Length', 'Content-Type', 'Content-Encoding', + if key in ('content-length', 'content-type', 'content-encoding', 'etag', 'last-modified'): self.assertTrue(key in headers) self.assertEquals(headers[key], val) @@ -455,9 +473,10 @@ resp = local_app(req.environ, local_app.app.do_start_response) self.assertEquals(local_app.app.response_args[0].split()[0], '206') - headers = dict(local_app.app.response_args[1]) - self.assertTrue('Content-Range' in headers) - self.assertTrue(headers['Content-Range'].startswith('bytes 0-3')) + headers = dict((k.lower(), v) for k, v in + local_app.app.response_args[1]) + self.assertTrue('content-range' in headers) + self.assertTrue(headers['content-range'].startswith('bytes=0-3')) def test_object_PUT_error(self): code = self._test_method_error(FakeAppObject, 'PUT', @@ -482,8 +501,9 @@ resp = local_app(req.environ, local_app.app.do_start_response) self.assertEquals(local_app.app.response_args[0].split()[0], '200') - headers = dict(local_app.app.response_args[1]) - self.assertEquals(headers['ETag'], + headers = dict((k.lower(), v) for k, v in + local_app.app.response_args[1]) + self.assertEquals(headers['etag'], "\"%s\"" % local_app.app.response_headers['etag']) def test_object_PUT_headers(self): @@ -601,11 +621,11 @@ local_app = swift3.filter_factory({})(app) req = Request.blank('/bucket/object?Signature=X&Expires=Y&' 'AWSAccessKeyId=Z', environ={'REQUEST_METHOD': 'GET'}) - req.date = datetime.now() + req.headers['Date'] = datetime.utcnow() req.content_type = 'text/plain' resp = local_app(req.environ, lambda *args: None) - self.assertEquals(app.req.headers['Authorization'], 'AWS Z:X') - self.assertEquals(app.req.headers['Date'], 'Y') + self.assertEquals(req.headers['Authorization'], 'AWS Z:X') + self.assertEquals(req.headers['Date'], 'Y') if __name__ == '__main__': unittest.main()