diff -Nru backblaze-b2-1.1.0/appveyor.yml backblaze-b2-1.3.6/appveyor.yml --- backblaze-b2-1.1.0/appveyor.yml 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/appveyor.yml 2018-08-21 22:00:18.000000000 +0000 @@ -9,75 +9,34 @@ matrix: - # Python 2.7.10 is the latest version and is not pre-installed. + # Python names come from: https://www.appveyor.com/docs/build-environment/#python - #- PYTHON: "C:\\Python27.10" - # PYTHON_VERSION: "2.7.10" - # PYTHON_ARCH: "32" + # Python 2.6 does not work any more. It gets errors installing arrow. + # I'm not going to take the time right now to figure out why. - #- PYTHON: "C:\\Python27.10-x64" - # PYTHON_VERSION: "2.7.10" - # PYTHON_ARCH: "64" - - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - # See: http://www.appveyor.com/docs/installed-software#python - - #- PYTHON: "C:\\Python27" - # PYTHON_VERSION: "2.7.x" # currently 2.7.9 - # PYTHON_ARCH: "32" - - #- PYTHON: "C:\\Python27-x64" - # PYTHON_VERSION: "2.7.x" # currently 2.7.9 - # PYTHON_ARCH: "64" - - #- PYTHON: "C:\\Python33" - # PYTHON_VERSION: "3.3.x" # currently 3.3.5 - # PYTHON_ARCH: "32" - - #- PYTHON: "C:\\Python33-x64" - # PYTHON_VERSION: "3.3.x" # currently 3.3.5 - # PYTHON_ARCH: "64" - - #- PYTHON: "C:\\Python34" - # PYTHON_VERSION: "3.4.x" # currently 3.4.3 - # PYTHON_ARCH: "32" - - #- PYTHON: "C:\\Python34-x64" - # PYTHON_VERSION: "3.4.x" # currently 3.4.3 - # PYTHON_ARCH: "64" - - # Python versions not pre-installed + - PYTHON: "C://Python27" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "32" - # Python 2.6.6 is the latest Python 2.6 with a Windows installer - # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 + - PYTHON: "C://Python27-x64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" - - PYTHON: "C:\\Python266" - PYTHON_VERSION: "2.6.6" + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5" PYTHON_ARCH: "32" - #- PYTHON: "C:\\Python266-x64" - # PYTHON_VERSION: "2.6.6" - # PYTHON_ARCH: "64" - - #- PYTHON: "C:\\Python35" - # PYTHON_VERSION: "3.5.0" - # PYTHON_ARCH: "32" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.0" + PYTHON_VERSION: "3.5" PYTHON_ARCH: "64" - # Major and minor releases (i.e x.0.0 and x.y.0) prior to 3.3.0 use - # a different naming scheme. + - PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6" + PYTHON_ARCH: "32" - #- PYTHON: "C:\\Python270" - # PYTHON_VERSION: "2.7.0" - # PYTHON_ARCH: "32" - - #- PYTHON: "C:\\Python270-x64" - # PYTHON_VERSION: "2.7.0" - # PYTHON_ARCH: "64" + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6" + PYTHON_ARCH: "64" install: - ECHO "Filesystem root:" @@ -101,7 +60,7 @@ # Upgrade to the latest version of pip to avoid it displaying warnings # about it being out of date. - - "pip install --disable-pip-version-check --user --upgrade pip" + - "python -m pip install --disable-pip-version-check --user --upgrade pip" # Install the build dependencies of the project. If some dependencies contain # compiled extensions and are not provided as pre-built wheel packages, diff -Nru backblaze-b2-1.1.0/b2/account_info/abstract.py backblaze-b2-1.3.6/b2/account_info/abstract.py --- backblaze-b2-1.1.0/b2/account_info/abstract.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/abstract.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/abstract.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -12,7 +12,8 @@ import six -from ..utils import B2TraceMetaAbstract, limit_trace_arguments +from b2.raw_api import ALL_CAPABILITIES +from b2.utils import B2TraceMetaAbstract, limit_trace_arguments @six.add_metaclass(B2TraceMetaAbstract) @@ -30,10 +31,22 @@ REALM_URLS = { 'production': 'https://api.backblazeb2.com', - 'dev': 'http://api.test.blaze:8180', + 'dev': 'http://api.backblazeb2.xyz:8180', 'staging': 'https://api.backblaze.net', } + # The 'allowed' structure to use for old account info that was saved without 'allowed'. + DEFAULT_ALLOWED = dict( + bucketId=None, + bucketName=None, + capabilities=ALL_CAPABILITIES, + namePrefix=None, + ) + + @classmethod + def all_capabilities(cls): + return cls.ALL_CAPABILITIES + @abstractmethod def clear(self): """ @@ -76,6 +89,10 @@ """ returns account_id or raises MissingAccountData exception """ @abstractmethod + def get_account_id_or_app_key_id(self): + """ returns the account id or key id used to authenticate """ + + @abstractmethod def get_account_auth_token(self): """ returns account_auth_token or raises MissingAccountData exception """ @@ -102,12 +119,83 @@ """ @abstractmethod + def get_allowed(self): + """ + An 'allowed' dict, as returned by b2_authorize_account. + Never None; for account info that was saved before 'allowed' existed, + returns DEFAULT_ALLOWED. + """ + @limit_trace_arguments(only=['self', 'api_url', 'download_url', 'minimum_part_size', 'realm']) def set_auth_data( - self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, - realm + self, + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, + allowed=None, + account_id_or_app_key_id=None, ): - pass + """ + Stores the results of b2_authorize_account. + + All of the information returned by b2_authorize_account is saved, because all of it is + needed by some command. + + The allowed structure is the one returned b2_authorize_account, with the addition of + a bucketName field. For keys with bucket restrictions, the name of the bucket is looked + up and stored, too. The console_tool does everything by bucket name, so it's convenient + to have the restricted bucket name handy. + """ + if allowed is None: + allowed = self.DEFAULT_ALLOWED + assert self.allowed_is_valid(allowed) + self._set_auth_data( + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, + allowed, + account_id_or_app_key_id, + ) + + @classmethod + def allowed_is_valid(cls, allowed): + """ + Makes sure that all of the required fields are present, and that + bucketId is set if bucketName is. + + If the bucketId is for a bucket that no longer exists, or the + capabilities do not allow listBuckets, then we won't have a bucketName. + """ + return ( + ('bucketId' in allowed) and ('bucketName' in allowed) and + ((allowed['bucketId'] is not None) or (allowed['bucketName'] is None)) and + ('capabilities' in allowed) and ('namePrefix' in allowed) + ) + + @abstractmethod + def _set_auth_data( + self, + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, + allowed, + account_id_or_app_key_id, + ): + """ + Stores the auth data. Can assume that 'allowed' is present and valid. + """ @abstractmethod def take_bucket_upload_url(self, bucket_id): diff -Nru backblaze-b2-1.1.0/b2/account_info/exception.py backblaze-b2-1.3.6/b2/account_info/exception.py --- backblaze-b2-1.1.0/b2/account_info/exception.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/exception.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/exception.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff -Nru backblaze-b2-1.1.0/b2/account_info/in_memory.py backblaze-b2-1.3.6/b2/account_info/in_memory.py --- backblaze-b2-1.1.0/b2/account_info/in_memory.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/in_memory.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/in_memory.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -29,25 +29,46 @@ class InMemoryAccountInfo(UrlPoolAccountInfo): def __init__(self, *args, **kwargs): - self.clear() super(InMemoryAccountInfo, self).__init__(*args, **kwargs) + self._clear_in_memory_account_fields() def clear(self): - self.set_auth_data(None, None, None, None, None, None, None) - self._buckets = {} + self._clear_in_memory_account_fields() return super(InMemoryAccountInfo, self).clear() - def set_auth_data( - self, account_id, auth_token, api_url, download_url, minimum_part_size, application_key, - realm + def _clear_in_memory_account_fields(self): + self._account_id = None + self._account_id_or_app_key_id = None + self._allowed = None + self._api_url = None + self._application_key = None + self._auth_token = None + self._buckets = {} + self._download_url = None + self._minimum_part_size = None + self._realm = None + + def _set_auth_data( + self, + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, + allowed, + account_id_or_app_key_id, ): self._account_id = account_id + self._account_id_or_app_key_id = account_id_or_app_key_id self._auth_token = auth_token self._api_url = api_url self._download_url = download_url self._minimum_part_size = minimum_part_size self._application_key = application_key self._realm = realm + self._allowed = allowed def refresh_entire_bucket_name_cache(self, name_id_iterable): self._buckets = dict(name_id_iterable) @@ -67,6 +88,10 @@ return self._account_id @_raise_missing_if_result_is_none + def get_account_id_or_app_key_id(self): + return self._account_id_or_app_key_id + + @_raise_missing_if_result_is_none def get_account_auth_token(self): return self._auth_token @@ -89,3 +114,7 @@ @_raise_missing_if_result_is_none def get_realm(self): return self._realm + + @_raise_missing_if_result_is_none + def get_allowed(self): + return self._allowed diff -Nru backblaze-b2-1.1.0/b2/account_info/sqlite_account_info.py backblaze-b2-1.3.6/b2/account_info/sqlite_account_info.py --- backblaze-b2-1.1.0/b2/account_info/sqlite_account_info.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/sqlite_account_info.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/sqlite_account_info.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -32,9 +32,16 @@ """ Stores account information in an sqlite database, which is used to manage concurrent access to the data. + + The 'update_done' table tracks the schema updates that have been + completed. """ - def __init__(self, file_name=None): + def __init__(self, file_name=None, last_upgrade_to_run=None): + """ + :param file_name: The sqlite file to use; overrides the default. + :param last_upgrade_to_run: For testing only, override the auto-update on the db. + """ self.thread_local = threading.local() user_account_info_path = file_name or os.environ.get( B2_ACCOUNT_INFO_ENV_VAR, B2_ACCOUNT_INFO_DEFAULT_FILE @@ -42,23 +49,23 @@ self.filename = file_name or os.path.expanduser(user_account_info_path) self._validate_database() with self._get_connection() as conn: - self._create_tables(conn) + self._create_tables(conn, last_upgrade_to_run) super(SqliteAccountInfo, self).__init__() - def _validate_database(self): + def _validate_database(self, last_upgrade_to_run=None): """ Makes sure that the database is openable. Removes the file if it's not. """ # If there is no file there, that's fine. It will get created when # we connect. if not os.path.exists(self.filename): - self._create_database() + self._create_database(last_upgrade_to_run) return # If we can connect to the database, and do anything, then all is good. try: with self._connect() as conn: - self._create_tables(conn) + self._create_tables(conn, last_upgrade_to_run) return except sqlite3.DatabaseError: pass # fall through to next case @@ -76,10 +83,10 @@ # remove the json file os.unlink(self.filename) # create a database - self._create_database() + self._create_database(last_upgrade_to_run) # add the data from the JSON file with self._connect() as conn: - self._create_tables(conn) + self._create_tables(conn, last_upgrade_to_run) insert_statement = """ INSERT INTO account (account_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm) @@ -108,7 +115,7 @@ def _connect(self): return sqlite3.connect(self.filename, isolation_level='EXCLUSIVE') - def _create_database(self): + def _create_database(self, last_upgrade_to_run): """ Makes sure that the database is created and sets the file permissions. This should be done before storing any sensitive data in it. @@ -117,14 +124,22 @@ conn = self._connect() try: with conn: - self._create_tables(conn) + self._create_tables(conn, last_upgrade_to_run) finally: conn.close() # Set the file permissions os.chmod(self.filename, stat.S_IRUSR | stat.S_IWUSR) - def _create_tables(self, conn): + def _create_tables(self, conn, last_upgrade_to_run): + conn.execute( + """ + CREATE TABLE IF NOT EXISTS + update_done ( + update_number INT NOT NULL + ); + """ + ) conn.execute( """ CREATE TABLE IF NOT EXISTS @@ -161,6 +176,35 @@ ); """ ) + # By default, we run all the upgrades + last_upgrade_to_run = 2 if last_upgrade_to_run is None else last_upgrade_to_run + # Add the 'allowed' column if it hasn't been yet. + if 1 <= last_upgrade_to_run: + self._ensure_update(1, 'ALTER TABLE account ADD COLUMN allowed TEXT;') + # Add the 'account_id_or_app_key_id' column if it hasn't been yet + if 2 <= last_upgrade_to_run: + self._ensure_update(2, 'ALTER TABLE account ADD COLUMN account_id_or_app_key_id TEXT;') + + def _ensure_update(self, update_number, update_command): + """ + Runs the update with the given number if it hasn't been done yet. + + Does the update and stores the number as a single transaction, + so they will always be in sync. + """ + with self._get_connection() as conn: + conn.execute('BEGIN') + cursor = conn.execute( + 'SELECT COUNT(*) AS count FROM update_done WHERE update_number = ?;', + (update_number,) + ) + update_count = cursor.fetchone()[0] + assert update_count in [0, 1] + if update_count == 0: + conn.execute(update_command) + conn.execute( + 'INSERT INTO update_done (update_number) VALUES (?);', (update_number,) + ) def clear(self): with self._get_connection() as conn: @@ -168,9 +212,52 @@ conn.execute('DELETE FROM bucket;') conn.execute('DELETE FROM bucket_upload_url;') - def set_auth_data( - self, account_id, account_auth_token, api_url, download_url, minimum_part_size, - application_key, realm + def _set_auth_data( + self, + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, + allowed, + account_id_or_app_key_id, + ): + assert self.allowed_is_valid(allowed) + with self._get_connection() as conn: + conn.execute('DELETE FROM account;') + conn.execute('DELETE FROM bucket;') + conn.execute('DELETE FROM bucket_upload_url;') + insert_statement = """ + INSERT INTO account + (account_id, account_id_or_app_key_id, application_key, account_auth_token, api_url, download_url, minimum_part_size, realm, allowed) + values (?, ?, ?, ?, ?, ?, ?, ?, ?); + """ + + conn.execute( + insert_statement, ( + account_id, + account_id_or_app_key_id, + application_key, + auth_token, + api_url, + download_url, + minimum_part_size, + realm, + json.dumps(allowed), + ) + ) + + def set_auth_data_with_schema_0_for_test( + self, + account_id, + auth_token, + api_url, + download_url, + minimum_part_size, + application_key, + realm, ): with self._get_connection() as conn: conn.execute('DELETE FROM account;') @@ -184,8 +271,13 @@ conn.execute( insert_statement, ( - account_id, application_key, account_auth_token, api_url, download_url, - minimum_part_size, realm + account_id, + application_key, + auth_token, + api_url, + download_url, + minimum_part_size, + realm, ) ) @@ -195,6 +287,16 @@ def get_account_id(self): return self._get_account_info_or_raise('account_id') + def get_account_id_or_app_key_id(self): + """ + The 'account_id_or_app_key_id' column was not in the original schema, so it may be NULL. + """ + result = self._get_account_info_or_raise('account_id_or_app_key_id') + if result is None: + return self.get_account_id() + else: + return result + def get_api_url(self): return self._get_account_info_or_raise('api_url') @@ -210,6 +312,16 @@ def get_minimum_part_size(self): return self._get_account_info_or_raise('minimum_part_size') + def get_allowed(self): + """ + The 'allowed" column was not in the original schema, so it may be NULL. + """ + allowed_json = self._get_account_info_or_raise('allowed') + if allowed_json is None: + return self.DEFAULT_ALLOWED + else: + return json.loads(allowed_json) + def _get_account_info_or_raise(self, column_name): try: with self._get_connection() as conn: diff -Nru backblaze-b2-1.1.0/b2/account_info/test_upload_url_concurrency.py backblaze-b2-1.3.6/b2/account_info/test_upload_url_concurrency.py --- backblaze-b2-1.1.0/b2/account_info/test_upload_url_concurrency.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/test_upload_url_concurrency.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/test_upload_conncurrency.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff -Nru backblaze-b2-1.1.0/b2/account_info/upload_url_pool.py backblaze-b2-1.3.6/b2/account_info/upload_url_pool.py --- backblaze-b2-1.1.0/b2/account_info/upload_url_pool.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/account_info/upload_url_pool.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/account_info/upload_url_pool.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # diff -Nru backblaze-b2-1.1.0/b2/api.py backblaze-b2-1.3.6/b2/api.py --- backblaze-b2-1.1.0/b2/api.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/api.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/api.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -15,11 +15,10 @@ from .b2http import B2Http from .bucket import Bucket, BucketFactory from .cache import AuthInfoCache, DummyCache -from .download_dest import DownloadDestProgressWrapper -from .exception import NonExistentBucket +from .transferer import Transferer +from .exception import NonExistentBucket, RestrictedBucket from .file_version import FileVersionInfoFactory, FileIdAndName from .part import PartFactory -from .progress import DoNothingProgressListener from .raw_api import B2RawApi from .session import B2Session from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments @@ -73,6 +72,7 @@ if cache is None: cache = AuthInfoCache(account_info) self.session = B2Session(self, self.raw_api) + self.transferer = Transferer(self.session, account_info) self.account_info = account_info if cache is None: cache = DummyCache() @@ -103,7 +103,7 @@ try: self.authorize_account( self.account_info.get_realm(), - self.account_info.get_account_id(), + self.account_info.get_account_id_or_app_key_id(), self.account_info.get_application_key(), ) except MissingAccountData: @@ -111,18 +111,49 @@ return True @limit_trace_arguments(only=('self', 'realm')) - def authorize_account(self, realm, account_id, application_key): + def authorize_account(self, realm, account_id_or_key_id, application_key): + # Clean up any previous account info if it was for a different account. try: old_account_id = self.account_info.get_account_id() old_realm = self.account_info.get_realm() - if account_id != old_account_id or realm != old_realm: + if account_id_or_key_id != old_account_id or realm != old_realm: self.cache.clear() except MissingAccountData: self.cache.clear() + # Authorize realm_url = self.account_info.REALM_URLS[realm] - response = self.raw_api.authorize_account(realm_url, account_id, application_key) + response = self.raw_api.authorize_account(realm_url, account_id_or_key_id, application_key) + allowed = response['allowed'] + # If there is a bucket restriction, get the name of the bucket. + # And, if we have a list of the one allowed bucket, go ahead and + # save it. + if 'listBuckets' in allowed['capabilities']: + if allowed['bucketId'] is not None: + allowed_bucket_response = self.raw_api.list_buckets( + api_url=response['apiUrl'], + account_auth_token=response['authorizationToken'], + account_id=response['accountId'], + bucket_id=allowed['bucketId'], + ) + allowed_bucket_list = allowed_bucket_response['buckets'] + allowed_bucket_count = len(allowed_bucket_list) + assert allowed_bucket_count in [0, 1] + if allowed_bucket_count == 0: + # bucket has been deleted since the key was made + allowed['bucketName'] = None + else: + allowed['bucketName'] = allowed_bucket_list[0]['bucketName'] + self.cache.set_bucket_name_cache( + BucketFactory.from_api_response(self, allowed_bucket_response) + ) + else: + allowed['bucketName'] = None + else: + allowed['bucketName'] = None + + # Store the auth data self.account_info.set_auth_data( response['accountId'], response['authorizationToken'], @@ -131,6 +162,8 @@ response['minimumPartSize'], application_key, realm, + allowed, + account_id_or_key_id, ) def get_account_id(self): @@ -162,33 +195,38 @@ return bucket def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): - progress_listener = progress_listener or DoNothingProgressListener() - self.session.download_file_by_id( + url = self.session.get_download_url_by_id( file_id, - DownloadDestProgressWrapper(download_dest, progress_listener), url_factory=self.account_info.get_download_url, - range_=range_, ) - progress_listener.close() + return self.transferer.download_file_from_url(url, download_dest, progress_listener, range_) def get_bucket_by_id(self, bucket_id): return Bucket(self, bucket_id) def get_bucket_by_name(self, bucket_name): """ - Returns the bucket_id for the given bucket_name. + Returns the Bucket for the given bucket_name. - If we don't already know it from the cache, try fetching it from - the B2 service. + :param bucket_name: The name of the bucket to return. + :return: a Bucket object + :raises NonExistentBucket: if the bucket does not exist in the account """ - # If we can get it from the stored info, do that. + # Give a useful warning if the current application key does not + # allow access to the named bucket. + self.check_bucket_restrictions(bucket_name) + + # First, try the cache. id_ = self.cache.get_bucket_id_or_none_from_bucket_name(bucket_name) if id_ is not None: return Bucket(self, id_, name=bucket_name) - for bucket in self.list_buckets(): - if bucket.name == bucket_name: - return bucket + # Second, ask the service + for bucket in self.list_buckets(bucket_name=bucket_name): + assert bucket.name == bucket_name + return bucket + + # There is no such bucket. raise NonExistentBucket(bucket_name) def delete_bucket(self, bucket): @@ -201,17 +239,38 @@ account_id = self.account_info.get_account_id() return self.session.delete_bucket(account_id, bucket.id_) - def list_buckets(self): - """ - Calls b2_list_buckets and returns the JSON for *all* buckets. + def list_buckets(self, bucket_name=None): """ - account_id = self.account_info.get_account_id() + Calls b2_list_buckets and returns a list of buckets. + + When no bucket name is specified, returns *all* of the buckets + in the account. When a bucket name is given, returns just that + bucket. When authorized with an application key restricted to + one bucket, you must specify the bucket name, or the request + will be unauthorized. + + :param bucket_name: Optional: the name of the one bucket to return. + :return: A list of Bucket objects. + """ + # Give a useful warning if the current application key does not + # allow access to the named bucket. + self.check_bucket_restrictions(bucket_name) - response = self.session.list_buckets(account_id) + account_id = self.account_info.get_account_id() + self.check_bucket_restrictions(bucket_name) + response = self.session.list_buckets(account_id, bucket_name=bucket_name) buckets = BucketFactory.from_api_response(self, response) - self.cache.set_bucket_name_cache(buckets) + if bucket_name is not None: + # If a bucket_name is specified we don't clear the cache because the other buckets could still + # be valid. So we save the one bucket returned from the list_buckets call. + for bucket in buckets: + self.cache.save_bucket(bucket) + else: + # Otherwise we want to clear the cache and save the buckets returned from list_buckets + # since we just got a new list of all the buckets for this account. + self.cache.set_bucket_name_cache(buckets) return buckets def list_parts(self, file_id, start_part_number=None, batch_size=None): @@ -252,11 +311,59 @@ """ Returns a URL to download the given file by name. """ + self.check_bucket_restrictions(bucket_name) return '%s/file/%s/%s' % ( self.account_info.get_download_url(), bucket_name, b2_url_encode(file_name) ) + # keys + def create_key( + self, capabilities, key_name, valid_duration_seconds=None, bucket_id=None, name_prefix=None + ): + account_id = self.account_info.get_account_id() + + response = self.session.create_key( + account_id, + capabilities=capabilities, + key_name=key_name, + valid_duration_seconds=valid_duration_seconds, + bucket_id=bucket_id, + name_prefix=name_prefix + ) + + assert set(response['capabilities']) == set(capabilities) + assert response['keyName'] == key_name + + return response + + def delete_key(self, application_key_id): + + response = self.session.delete_key(application_key_id=application_key_id) + return response + + def list_keys(self, start_application_key_id=None): + account_id = self.account_info.get_account_id() + + return self.session.list_keys( + account_id, max_key_count=1000, start_application_key_id=start_application_key_id + ) + # other def get_file_info(self, file_id): """ legacy interface which just returns whatever remote API returns """ return self.session.get_file_info(file_id) + + def check_bucket_restrictions(self, bucket_name): + """ + Checks to see if the allowed field from authorize-account + has a bucket restriction. + + If it does, does the bucket_name for a given api call match that. + If not it raises a RestrictedBucket error. + """ + allowed = self.account_info.get_allowed() + allowed_bucket_name = allowed['bucketName'] + + if allowed_bucket_name is not None: + if allowed_bucket_name != bucket_name: + raise RestrictedBucket(allowed_bucket_name) diff -Nru backblaze-b2-1.1.0/b2/b2http.py backblaze-b2-1.3.6/b2/b2http.py --- backblaze-b2-1.1.0/b2/b2http.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/b2http.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/b2http.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -19,7 +19,10 @@ import six import time -from .exception import B2Error, BadDateFormat, BrokenPipe, B2ConnectionError, B2RequestTimeout, ClockSkew, ConnectionReset, interpret_b2_error, UnknownError, UnknownHost +from .exception import ( + B2Error, BadDateFormat, BrokenPipe, B2ConnectionError, B2RequestTimeout, ClockSkew, + ConnectionReset, interpret_b2_error, UnknownError, UnknownHost +) from .version import USER_AGENT from six.moves import range @@ -386,7 +389,7 @@ try: b2_http.post_json_return_json('https://unknown.backblazeb2.com', {}, {}) assert False, 'should have failed with unknown host' - except UnknownHost as e: + except UnknownHost: pass # Broken pipe @@ -395,7 +398,7 @@ data = six.BytesIO(six.b(chr(0)) * 10000000) b2_http.post_content_return_json('https://api.backblazeb2.com/bad_url', {}, data) assert False, 'should have failed with broken pipe' - except BrokenPipe as e: + except BrokenPipe: pass # Generic connection error @@ -404,5 +407,5 @@ with b2_http.get_content('https://www.backblazeb2.com:80/bad_url', {}) as response: assert False, 'should have failed with connection error' response.iter_content() # make pyflakes happy - except B2ConnectionError as e: + except B2ConnectionError: pass diff -Nru backblaze-b2-1.1.0/b2/bounded_queue_executor.py backblaze-b2-1.3.6/b2/bounded_queue_executor.py --- backblaze-b2-1.1.0/b2/bounded_queue_executor.py 1970-01-01 00:00:00.000000000 +0000 +++ backblaze-b2-1.3.6/b2/bounded_queue_executor.py 2018-08-21 22:00:18.000000000 +0000 @@ -0,0 +1,55 @@ +###################################################################### +# +# File: b2/bounded_queue_executor.py +# +# Copyright 2018 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import threading + + +class BoundedQueueExecutor(object): + """ + Wraps a futures.Executor and limits the number of requests that + can be queued at once. Requests to submit() tasks block until + there is room in the queue. + + The number of available slots in the queue is tracked with a + semaphore that is acquired before queueing an action, and + released when an action finishes. + + Counts the number of exceptions thrown by tasks, and makes them + available from get_num_exceptions() after shutting down. + """ + + def __init__(self, executor, queue_limit): + self.executor = executor + self.semaphore = threading.Semaphore(queue_limit) + self.num_exceptions = 0 + + def submit(self, fcn, *args, **kwargs): + # Wait until there is room in the queue. + self.semaphore.acquire() + + # Wrap the action in a function that will release + # the semaphore after it runs. + def run_it(): + try: + return fcn(*args, **kwargs) + except Exception: + self.num_exceptions += 1 + raise + finally: + self.semaphore.release() + + # Submit the wrapped action. + return self.executor.submit(run_it) + + def shutdown(self): + self.executor.shutdown() + + def get_num_exceptions(self): + return self.num_exceptions diff -Nru backblaze-b2-1.1.0/b2/bucket.py backblaze-b2-1.3.6/b2/bucket.py --- backblaze-b2-1.1.0/b2/bucket.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/bucket.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/bucket.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -12,12 +12,11 @@ import six import threading -from .download_dest import DownloadDestProgressWrapper from .exception import ( AlreadyFailed, B2Error, MaxFileSizeExceeded, MaxRetriesExceeded, UnrecognizedBucketType ) from .file_version import FileVersionInfoFactory -from .progress import DoNothingProgressListener, AbstractProgressListener, RangeOfInputStream, StreamWithProgress, StreamWithHash +from .progress import DoNothingProgressListener, AbstractProgressListener, RangeOfInputStream, ReadingStreamWithProgress, StreamWithHash from .unfinished_large_file import UnfinishedLargeFile from .upload_source import UploadSourceBytes, UploadSourceLocalFile from .utils import b2_url_encode, choose_part_ranges, hex_sha1_of_stream, interruptible_get_result, validate_b2_file_name @@ -154,18 +153,20 @@ return self.api.cancel_large_file(file_id) def download_file_by_id(self, file_id, download_dest, progress_listener=None, range_=None): - self.api.download_file_by_id(file_id, download_dest, progress_listener, range_=range_) + # download_file_by_id actually belongs in B2Api, not in Bucket, we just provide a convenient redirect here + return self.api.download_file_by_id( + file_id, download_dest, progress_listener, range_=range_ + ) def download_file_by_name(self, file_name, download_dest, progress_listener=None, range_=None): - progress_listener = progress_listener or DoNothingProgressListener() - self.api.session.download_file_by_name( + url = self.api.session.get_download_url_by_name( self.name, file_name, - DownloadDestProgressWrapper(download_dest, progress_listener), url_factory=self.api.account_info.get_download_url, - range_=range_, ) - progress_listener.close() + return self.api.transferer.download_file_from_url( + url, download_dest, progress_listener, range_ + ) def get_download_authorization(self, file_name_prefix, valid_duration_in_seconds): response = self.api.session.get_download_authorization( @@ -398,33 +399,33 @@ content_length = upload_source.get_content_length() upload_url = None exception_info_list = [] - for _ in six.moves.xrange(self.MAX_UPLOAD_ATTEMPTS): - # refresh upload data in every attempt to work around a "busy storage pod" - upload_url, upload_auth_token = self._get_upload_data() - - try: - with upload_source.open() as file: - progress_listener.set_total_bytes(content_length) - input_stream = StreamWithProgress(file, progress_listener) - hashing_stream = StreamWithHash(input_stream) - length_with_hash = content_length + hashing_stream.hash_size() - response = self.api.raw_api.upload_file( - upload_url, upload_auth_token, file_name, length_with_hash, content_type, - HEX_DIGITS_AT_END, file_info, hashing_stream - ) - assert hashing_stream.hash == response['contentSha1'] - self.api.account_info.put_bucket_upload_url( - self.id_, upload_url, upload_auth_token - ) - progress_listener.close() - return FileVersionInfoFactory.from_api_response(response) - - except B2Error as e: - logger.exception('error when uploading, upload_url was %s', upload_url) - if not e.should_retry_upload(): - raise - exception_info_list.append(e) - self.api.account_info.clear_bucket_upload_data(self.id_) + progress_listener.set_total_bytes(content_length) + with progress_listener: + for _ in six.moves.xrange(self.MAX_UPLOAD_ATTEMPTS): + # refresh upload data in every attempt to work around a "busy storage pod" + upload_url, upload_auth_token = self._get_upload_data() + + try: + with upload_source.open() as file: + input_stream = ReadingStreamWithProgress(file, progress_listener) + hashing_stream = StreamWithHash(input_stream) + length_with_hash = content_length + hashing_stream.hash_size() + response = self.api.raw_api.upload_file( + upload_url, upload_auth_token, file_name, length_with_hash, + content_type, HEX_DIGITS_AT_END, file_info, hashing_stream + ) + assert hashing_stream.hash == response['contentSha1'] + self.api.account_info.put_bucket_upload_url( + self.id_, upload_url, upload_auth_token + ) + return FileVersionInfoFactory.from_api_response(response) + + except B2Error as e: + logger.exception('error when uploading, upload_url was %s', upload_url) + if not e.should_retry_upload(): + raise + exception_info_list.append(e) + self.api.account_info.clear_bucket_upload_data(self.id_) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_info_list) @@ -438,7 +439,6 @@ # Set up the progress reporting for the parts progress_listener.set_total_bytes(content_length) - large_file_upload_state = LargeFileUploadState(progress_listener) # Select the part boundaries part_ranges = choose_part_ranges(content_length, minimum_part_size) @@ -453,27 +453,28 @@ unfinished_file = self.start_large_file(file_name, content_type, file_info) file_id = unfinished_file.file_id - # Tell the executor to upload each of the parts - part_futures = [ - self.api.get_thread_pool().submit( - self._upload_part, - file_id, - part_index + 1, # part number - part_range, - upload_source, - large_file_upload_state, - finished_parts - ) for (part_index, part_range) in enumerate(part_ranges) - ] - - # Collect the sha1 checksums of the parts as the uploads finish. - # If any of them raised an exception, that same exception will - # be raised here by result() - part_sha1_array = [interruptible_get_result(f)['contentSha1'] for f in part_futures] + with progress_listener: + large_file_upload_state = LargeFileUploadState(progress_listener) + # Tell the executor to upload each of the parts + part_futures = [ + self.api.get_thread_pool().submit( + self._upload_part, + file_id, + part_index + 1, # part number + part_range, + upload_source, + large_file_upload_state, + finished_parts + ) for (part_index, part_range) in enumerate(part_ranges) + ] + + # Collect the sha1 checksums of the parts as the uploads finish. + # If any of them raised an exception, that same exception will + # be raised here by result() + part_sha1_array = [interruptible_get_result(f)['contentSha1'] for f in part_futures] # Finish the large file response = self.api.session.finish_large_file(file_id, part_sha1_array) - progress_listener.close() return FileVersionInfoFactory.from_api_response(response) def _find_unfinished_file(self, upload_source, file_name, file_info, part_ranges): @@ -550,7 +551,7 @@ offset, content_length = part_range file.seek(offset) range_stream = RangeOfInputStream(file, offset, content_length) - input_stream = StreamWithProgress(range_stream, part_progress_listener) + input_stream = ReadingStreamWithProgress(range_stream, part_progress_listener) hashing_stream = StreamWithHash(input_stream) length_with_hash = content_length + hashing_stream.hash_size() response = self.api.raw_api.upload_part( diff -Nru backblaze-b2-1.1.0/b2/cache.py backblaze-b2-1.3.6/b2/cache.py --- backblaze-b2-1.1.0/b2/cache.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/cache.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/cache.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -23,6 +23,10 @@ pass @abstractmethod + def get_bucket_name_or_none_from_allowed(self): + pass + + @abstractmethod def save_bucket(self, bucket): pass @@ -40,6 +44,9 @@ def get_bucket_id_or_none_from_bucket_name(self, name): return None + def get_bucket_name_or_none_from_allowed(self): + return None + def save_bucket(self, bucket): pass @@ -52,10 +59,14 @@ def __init__(self): self.name_id_map = {} + self.bucket_name = '' def get_bucket_id_or_none_from_bucket_name(self, name): return self.name_id_map.get(name) + def get_bucket_name_or_none_from_allowed(self): + return self.bucket_name + def save_bucket(self, bucket): self.name_id_map[bucket.name] = bucket.id_ @@ -72,6 +83,9 @@ def get_bucket_id_or_none_from_bucket_name(self, name): return self.info.get_bucket_id_or_none_from_bucket_name(name) + def get_bucket_name_or_none_from_allowed(self): + return self.info.get_bucket_name_or_none_from_allowed() + def save_bucket(self, bucket): self.info.save_bucket(bucket) diff -Nru backblaze-b2-1.1.0/b2/console_tool.py backblaze-b2-1.3.6/b2/console_tool.py --- backblaze-b2-1.1.0/b2/console_tool.py 2017-12-01 01:55:23.000000000 +0000 +++ backblaze-b2-1.3.6/b2/console_tool.py 2018-08-21 22:00:18.000000000 +0000 @@ -2,7 +2,7 @@ # # File: b2/console_tool.py # -# Copyright 2016 Backblaze Inc. All Rights Reserved. +# Copyright 2018 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # @@ -10,6 +10,9 @@ from __future__ import absolute_import, print_function +import copy +import datetime +import functools import getpass import json import locale @@ -30,14 +33,15 @@ from .account_info.test_upload_url_concurrency import test_upload_url_concurrency from .account_info.exception import (MissingAccountData) from .api import (B2Api) -from .b2http import (test_http, B2Http) +from .b2http import (test_http) from .cache import (AuthInfoCache) from .download_dest import (DownloadDestLocalFile) from .exception import (B2Error, BadFileInfo) +from .sync.scan_policies import ScanPoliciesManager from .file_version import (FileVersionInfo) from .parse_args import parse_arg_list from .progress import (make_progress_listener) -from .raw_api import (SRC_LAST_MODIFIED_MILLIS, B2RawApi, test_raw_api) +from .raw_api import (SRC_LAST_MODIFIED_MILLIS, test_raw_api) from .sync import parse_sync_folder, sync_folders from .utils import (current_time_millis, set_shutting_down) from .version import (VERSION) @@ -76,6 +80,20 @@ return s[0].lower() + ''.join(c if c.islower() else '-' + c.lower() for c in s[1:]) +def parse_comma_separated_list(s): + return [word.strip() for word in s.split(',')] + + +def apply_or_none(fcn, value): + """ + If the value is None, return None, otherwise return the result of applying the function to it. + """ + if value is None: + return None + else: + return fcn(value) + + class Command(object): """ Base class for commands. Has basic argument parsing and printing. @@ -190,19 +208,26 @@ class AuthorizeAccount(Command): """ - b2 authorize-account [] [] + b2 authorize-account [] [] - Prompts for Backblaze accountID and applicationKey (unless they are given - on the command line). + Prompts for Backblaze accountID and applicationKey (unless they + are given on the command line). - The account ID is a 12-digit hex number that you can get from - your account page on backblaze.com. + You can authorize with either the master application key or + a normal application key. - The application key is a 40-digit hex number that you can get from - your account page on backblaze.com. + To use the master application key, provide the account ID and + application key from the "B2 Cloud Storage Buckets" page on + the web site: https://secure.backblaze.com/b2_buckets.htm + + To use a normal application key, created with the create-key + command or on the web site, provide the application key ID + and the application key itself. Stores an account auth token in {B2_ACCOUNT_INFO_DEFAULT_FILE} by default, or the file specified by the {B2_ACCOUNT_INFO_ENV_VAR} environment variable. + + Requires capability: listBuckets """ OPTION_FLAGS = ['dev', 'staging'] # undocumented @@ -231,6 +256,25 @@ try: self.api.authorize_account(realm, args.accountId, args.applicationKey) + + allowed = self.api.account_info.get_allowed() + if 'listBuckets' not in allowed['capabilities']: + logger.error( + 'ConsoleTool cannot work with a bucket-restricted key and no listBuckets capability' + ) + self._print_stderr( + 'ERROR: application key has no listBuckets capability, which is required for the b2 command-line tool' + ) + self.api.account_info.clear() + return 1 + if allowed['bucketId'] is not None and allowed['bucketName'] is None: + logger.error('ConsoleTool has bucket-restricted key and the bucket does not exist') + self._print_stderr( + "ERROR: application key is restricted to bucket id '%s', which no longer exists" + % (allowed['bucketId'],) + ) + self.api.account_info.clear() + return 1 return 0 except B2Error as e: logger.exception('ConsoleTool account authorization error') @@ -245,6 +289,8 @@ Lists all large files that have been started but not finsished and cancels them. Any parts that have been uploaded will be deleted. + + Requires capability: writeFiles """ REQUIRED = ['bucketName'] @@ -260,6 +306,13 @@ class CancelLargeFile(Command): """ b2 cancel-large-file + + Cancels a large file upload. Used to undo a start-large-file. + + Cannot be used once the file is finished. After finishing, + using delete-file-version to delete the large file. + + Requires capability: writeFiles """ REQUIRED = ['fileId'] @@ -291,6 +344,8 @@ Optionally stores bucket info, CORS rules and lifecycle rules with the bucket. These can be given as JSON on the command line. + + Requires capability: writeBuckets """ REQUIRED = ['bucketName', 'bucketType'] @@ -311,11 +366,65 @@ return 0 +class CreateKey(Command): + """ + b2 create-key [--duration ] [--bucket ] [--namePrefix ] + + Creates a new application key. Prints the application key information. This is the only + time the application key itself will be returned. Listing application keys will show + their IDs, but not the secret keys. + + The capabilities are passed in as a comma-separated list, like "readFiles,writeFiles". + + The 'validDurationSeconds' is the length of time the new application key will exist. + When the time expires the key will disappear and will no longer be usable. If not + specified, the key will not expire. + + The 'bucketName' is the name of a bucket in the account. When specified, the key + will only allow access to that bucket. + + The 'namePrefix' restricts file access to files whose names start with the prefix. + + The output is the new application key ID, followed by the application key itself. + The two values returned are the two that you pass to authorize-account to use the key. + + Requires capability: writeKeys + """ + + REQUIRED = ['keyName', 'capabilities'] + + OPTION_ARGS = ['bucket', 'namePrefix', 'duration'] + + ARG_PARSER = {'capabilities': parse_comma_separated_list, 'duration': int} + + def run(self, args): + # Translate the bucket name into a bucketId + if args.bucket is None: + bucket_id_or_none = None + else: + bucket_id_or_none = self.api.get_bucket_by_name(args.bucket).id_ + + response = self.api.create_key( + capabilities=args.capabilities, + key_name=args.keyName, + valid_duration_seconds=args.duration, + bucket_id=bucket_id_or_none, + name_prefix=args.namePrefix + ) + + application_key_id = response['applicationKeyId'] + application_key = response['applicationKey'] + self._print(application_key_id + " " + application_key) + return 0 + + class DeleteBucket(Command): """ b2 delete-bucket Deletes the bucket with the given name. + + Requires capability: deleteBuckets """ REQUIRED = ['bucketName'] @@ -337,6 +446,8 @@ If you omit the fileName, it requires an initial query to B2 to get the file name, before making the call to delete the file. + + Requires capability: deleteFiles """ OPTIONAL_BEFORE = ['fileName'] @@ -347,6 +458,7 @@ file_name = args.fileName else: file_name = self._get_file_name_from_file_id(args.fileId) + file_info = self.api.delete_file_version(args.fileId, file_name) self._print(json.dumps(file_info.as_dict(), indent=2, sort_keys=True)) return 0 @@ -356,6 +468,23 @@ return file_info['fileName'] +class DeleteKey(Command): + """ + b2 delete-key + + Deletes the specified application key by its 'ID'. + + Requires capability: deleteKeys + """ + + REQUIRED = ['applicationKeyId'] + + def run(self, args): + response = self.api.delete_key(application_key_id=args.applicationKeyId) + self._print(response['applicationKeyId']) + return 0 + + class DownloadFileById(Command): """ b2 download-file-by-id [--noProgress] @@ -365,6 +494,8 @@ If the 'tqdm' library is installed, progress bar is displayed on stderr. Without it, simple text progress is printed. Use '--noProgress' to disable progress reporting. + + Requires capability: readFiles """ OPTION_FLAGS = ['noProgress'] @@ -383,6 +514,12 @@ b2 download-file-by-name [--noProgress] Downloads the given file, and stores it in the given local file. + + If the 'tqdm' library is installed, progress bar is displayed + on stderr. Without it, simple text progress is printed. + Use '--noProgress' to disable progress reporting. + + Requires capability: readFiles """ OPTION_FLAGS = ['noProgress'] @@ -401,13 +538,15 @@ """ b2 get-account-info - Shows the account ID, key, auth token, and URLs. + Shows the account ID, key, auth token, URLs, and what capabilities + the current application keys has. """ def run(self, args): account_info = self.api.account_info data = dict( accountId=account_info.get_account_id(), + allowed=account_info.get_allowed(), applicationKey=account_info.get_application_key(), accountAuthToken=account_info.get_account_auth_token(), apiUrl=account_info.get_api_url(), @@ -419,21 +558,54 @@ class GetBucket(Command): """ - b2 get-bucket + b2 get-bucket [--showSize] Prints all of the information about the bucket, including bucket info, CORS rules and lifecycle rules. + + If --showSize is specified, then display the number of files + (fileCount) in the bucket and the aggregate size of all files + (totalSize). Hidden files and hide markers are accounted for + in the reported number of files, and hidden files also + contribute toward the reported aggregate size, whereas hide + markers do not. Each version of a file counts as an individual + file, and its size contributes toward the aggregate size. + Analysis is recursive. Note that --showSize requires multiple + API calls, and will therefore incur additional latency, + computation, and Class C transactions. + + Requires capability: listBuckets """ + OPTION_FLAGS = ['showSize'] + REQUIRED = ['bucketName'] def run(self, args): # This always wants up-to-date info, so it does not use # the bucket cache. - for b in self.api.list_buckets(): - if b.name == args.bucketName: + for b in self.api.list_buckets(args.bucketName): + if not args.showSize: self._print(json.dumps(b.bucket_dict, indent=4, sort_keys=True)) return 0 + else: + # `files` is a generator. We don't want to collect all of the values from the + # generator, as there many be billions of files in a large bucket. + files = b.ls("", show_versions=True, recursive=True) + # `files` yields tuples of (file_version_info, folder_name). We don't care about + # `folder_name`, so just access the first slot of the tuple directly in the + # reducer. We can't ask a generator for its size, as the elements are yielded + # lazily, so we need to accumulate the count as we go. By using a tuple of + # (file count, total size), we can obtain the desired information very compactly + # and efficiently. + count_size_tuple = functools.reduce( + (lambda partial, f: (partial[0] + 1, partial[1] + f[0].size)), files, (0, 0) + ) + result = copy.copy(b.bucket_dict) + result['fileCount'] = count_size_tuple[0] + result['totalSize'] = count_size_tuple[1] + self._print(json.dumps(result, indent=4, sort_keys=True)) + return 0 self._print_stderr('bucket not found: ' + args.bucketName) return 1 @@ -443,6 +615,8 @@ b2 get-file-info Prints all of the information about the file, but not its contents. + + Requires capability: readFiles """ REQUIRED = ['fileId'] @@ -466,6 +640,8 @@ Only files that match that given prefix can be downloaded with the token. The prefix defaults to "", which matches all files in the bucket. + + Requires capability: shareFiles """ OPTION_ARGS = ['prefix', 'duration'] @@ -499,6 +675,8 @@ The token is valid for the duration specified, which defaults to 86400 seconds (one day). + + Requires capability: shareFiles """ OPTION_ARGS = ['duration'] @@ -545,6 +723,8 @@ b2 hide-file Uploads a new, hidden, version of the given file. + + Requires capability: writeFiles """ REQUIRED = ['bucketName', 'fileName'] @@ -567,6 +747,8 @@ and look like this: 98c960fd1cb4390c5e0f0519 allPublic my-bucket + + Requires capability: listBuckets """ def run(self, args): @@ -583,6 +765,8 @@ given point. This is a low-level operation that reports the raw JSON returned from the service. 'b2 ls' provides a higher- level view. + + Requires capability: listFiles """ REQUIRED = ['bucketName'] @@ -604,6 +788,8 @@ Lists the names of the files in a bucket, starting at the given point. + + Requires capability: listFiles """ REQUIRED = ['bucketName'] @@ -619,6 +805,97 @@ return 0 +class ListKeys(Command): + """ + b2 list-keys + + Lists the application keys for the current account. + + The columns in the output are: + - ID of the application key + - Name of the application key + - Name of the bucket the key is restricted to, or '-' for no restriction + - Date of expiration, or '-' + - Time of expiration, or '-' + - File name prefix, in single quotes + - Command-separated list of capabilities + + None of the values contain whitespace. + + For keys restricted to buckets that do not exist any more, the bucket name is + replaced with 'id=', because deleted buckets do not have names any + more. + + Requires capability: listKeys + """ + + OPTION_FLAGS = ['long'] + + def __init__(self, console_tool): + super(ListKeys, self).__init__(console_tool) + self.bucket_id_to_bucket_name = None + + def run(self, args): + # The first query doesn't pass in a starting key id + start_id = None + + # Keep querying until there are no more. + while True: + # Get some keys and print them + response = self.api.list_keys(start_id) + self.print_keys(response['keys'], args.long) + + # Are there more? If so, we'll set the start_id for the next time around. + next_id = response.get('nextApplicationKeyId') + if next_id is None: + break + else: + start_id = next_id + + return 0 + + def print_keys(self, keys_from_response, is_long_format): + if is_long_format: + format_str = "{keyId} {keyName:20s} {bucketName:20s} {dateStr:10s} {timeStr:8s} '{namePrefix}' {capabilities}" + else: + format_str = '{keyId} {keyName:20s}' + for key in keys_from_response: + timestamp_or_none = apply_or_none(int, key.get('expirationTimestamp')) + (date_str, time_str) = self.timestamp_display(timestamp_or_none) + key_str = format_str.format( + keyId=key['applicationKeyId'], + keyName=key['keyName'], + bucketName=self.bucket_display_name(key.get('bucketId')), + namePrefix=(key.get('namePrefix') or ''), + capabilities=','.join(key['capabilities']), + dateStr=date_str, + timeStr=time_str + ) + self._print(key_str) + + def bucket_display_name(self, bucket_id): + # Special case for no bucket ID + if bucket_id is None: + return '-' + + # Make sure we have the map + if self.bucket_id_to_bucket_name is None: + self.bucket_id_to_bucket_name = dict((b.id_, b.name) for b in self.api.list_buckets()) + + return self.bucket_id_to_bucket_name.get(bucket_id, 'id=' + bucket_id) + + def timestamp_display(self, timestamp_or_none): + """ + Returns a pair (date_str, time_str) for the given timestamp + """ + if timestamp_or_none is None: + return '-', '-' + else: + timestamp = timestamp_or_none + dt = datetime.datetime.utcfromtimestamp(timestamp / 1000) + return dt.strftime('%Y-%m-%d'), dt.strftime('%H:%M:%S') + + class ListParts(Command): """ b2 list-parts @@ -626,6 +903,8 @@ Lists all of the parts that have been uploaded for the given large file, which must be a file that was started but not finished or canceled. + + Requires capability: writeFiles """ REQUIRED = ['largeFileId'] @@ -643,6 +922,8 @@ Lists all of the large files in the bucket that were started, but not finished or canceled. + Requires capability: listFiles + """ REQUIRED = ['bucketName'] @@ -663,7 +944,7 @@ class Ls(Command): """ - b2 ls [--long] [--versions] [] + b2 ls [--long] [--versions] [--recursive] [] Using the file naming convention that "/" separates folder names from their contents, returns a list of the files @@ -678,9 +959,14 @@ The --version option shows all of versions of each file, not just the most recent. + + The --recursive option will descend into folders, and will show + only files, not folders. + + Requires capability: listFiles """ - OPTION_FLAGS = ['long', 'versions'] + OPTION_FLAGS = ['long', 'versions', 'recursive'] REQUIRED = ['bucketName'] @@ -695,7 +981,9 @@ prefix += '/' bucket = self.api.get_bucket_by_name(args.bucketName) - for file_version_info, folder_name in bucket.ls(prefix, args.versions): + for file_version_info, folder_name in bucket.ls( + prefix, show_versions=args.versions, recursive=args.recursive + ): if not args.long: self._print(folder_name or file_version_info.file_name) elif folder_name is not None: @@ -727,6 +1015,7 @@ [--compareVersions