diff -Nru nbformat-4.2.0/appveyor.yml nbformat-4.4.0/appveyor.yml --- nbformat-4.2.0/appveyor.yml 1970-01-01 00:00:00.000000000 +0000 +++ nbformat-4.4.0/appveyor.yml 2017-08-18 22:33:37.000000000 +0000 @@ -0,0 +1,31 @@ +# Do not build feature branch with open Pull Requests +skip_branch_with_pr: true + +# environment variables +environment: + matrix: + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5.x" + PYTHON_MAJOR: 3 + PYTHON_ARCH: "64" + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7.x" + PYTHON_MAJOR: 2 + PYTHON_ARCH: "32" + +# scripts that run after cloning repository +install: + # Ensure python scripts are from right version: + - 'SET "PATH=%PYTHON%\\Scripts;%PATH%"' + # Install our package: + - pip install codecov + - 'pip install --upgrade ".[test]"' + +build: off + +# to run your custom scripts instead of automatic tests +test_script: + - 'py.test -v --cov nbformat nbformat' + +on_success: + - codecov diff -Nru nbformat-4.2.0/debian/changelog nbformat-4.4.0/debian/changelog --- nbformat-4.2.0/debian/changelog 2016-12-04 11:56:02.000000000 +0000 +++ nbformat-4.4.0/debian/changelog 2017-09-03 14:36:44.000000000 +0000 @@ -1,3 +1,21 @@ +nbformat (4.4.0-1) unstable; urgency=medium + + [ Julien Puydt ] + * New upstream release. (Closes: #847393, #864700) + * Bump d/watch to version 4. + + [ Gordon Ball ] + * Bump Standards-Version to 4.1.0 (no changes required) + * Use python3 sphinx for documentation + * Add dependency on jupyter-core for jupyter-nbformat to ensure the script + can be invoked as "jupyter nbformat" + * Add myself to uploaders in d/control + * Use sphinxdoc:Built-Using for the documentation package + * New upstream release. + * Add Testsuite: autopkgtest-pkg-python + + -- Julien Puydt Sun, 03 Sep 2017 16:36:44 +0200 + nbformat (4.2.0-1) unstable; urgency=medium [ Gordon Ball ] diff -Nru nbformat-4.2.0/debian/control nbformat-4.4.0/debian/control --- nbformat-4.2.0/debian/control 2016-12-04 11:56:02.000000000 +0000 +++ nbformat-4.4.0/debian/control 2017-09-03 14:36:44.000000000 +0000 @@ -1,9 +1,9 @@ Source: nbformat Maintainer: Debian Python Modules Team -Uploaders: Julien Puydt +Uploaders: Julien Puydt , Gordon Ball Section: python Priority: optional -Standards-Version: 3.9.8 +Standards-Version: 4.1.0 Homepage: https://github.com/jupyter/nbformat Build-Depends: bc, debhelper (>= 10), @@ -13,10 +13,8 @@ python-jsonschema, python-jupyter-core, python-nose, - python-numpydoc, python-pytest, python-setuptools, - python-sphinx, python-testpath, python-traitlets, python3-all, @@ -24,6 +22,7 @@ python3-jsonschema, python3-jupyter-core, python3-nose, + python3-numpydoc, python3-pytest, python3-setuptools, python3-sphinx, @@ -33,6 +32,7 @@ X-Python3-Version: >= 3.3 Vcs-Git: https://anonscm.debian.org/git/python-modules/packages/nbformat.git Vcs-Browser: https://anonscm.debian.org/cgit/python-modules/packages/nbformat.git +Testsuite: autopkgtest-pkg-python Package: python-nbformat Architecture: all @@ -54,7 +54,10 @@ Package: jupyter-nbformat Architecture: all -Depends: ${misc:Depends}, ${python3:Depends}, python3-nbformat (= ${binary:Version}) +Depends: ${misc:Depends}, + ${python3:Depends}, + python3-nbformat (= ${binary:Version}), + jupyter-core Section: utils Description: Jupyter notebook format (tools) This software component contains the reference implementation of the Jupyter @@ -67,6 +70,7 @@ Architecture: all Multi-Arch: foreign Depends: ${misc:Depends}, ${sphinxdoc:Depends} +Built-Using: ${sphinxdoc:Built-Using} Description: Jupyter notebook format (documentation) This software component contains the reference implementation of the Jupyter notebook format, and Python APIs to work with notebooks. diff -Nru nbformat-4.2.0/debian/.git-dpm nbformat-4.4.0/debian/.git-dpm --- nbformat-4.2.0/debian/.git-dpm 2016-12-04 11:56:02.000000000 +0000 +++ nbformat-4.4.0/debian/.git-dpm 2017-09-03 14:36:44.000000000 +0000 @@ -1,11 +1,11 @@ # see git-dpm(1) from git-dpm package -1f14738fb23cb2f62ef42e3a73d2a706897886ad -1f14738fb23cb2f62ef42e3a73d2a706897886ad -abea36f71cf6aa39a0cf653f6476c033c72ec7f8 -abea36f71cf6aa39a0cf653f6476c033c72ec7f8 -nbformat_4.2.0.orig.tar.gz -fcb5030e3802eeda703333bb4885ae5ed460f88d -114714 +50cc84ceb4220c74dcc467e61a7e64f086e624ed +50cc84ceb4220c74dcc467e61a7e64f086e624ed +c08c300140cb82053678bdf4aebe622cc55bf768 +c08c300140cb82053678bdf4aebe622cc55bf768 +nbformat_4.4.0.orig.tar.gz +f740364f87f810de50c0550064853224971f1335 +119351 debianTag="debian/%e%v" patchedTag="patched/%e%v" upstreamTag="upstream/%e%u" diff -Nru nbformat-4.2.0/debian/patches/0001-Use-setuptools-for-all-targets-so-python-Depends-sub.patch nbformat-4.4.0/debian/patches/0001-Use-setuptools-for-all-targets-so-python-Depends-sub.patch --- nbformat-4.2.0/debian/patches/0001-Use-setuptools-for-all-targets-so-python-Depends-sub.patch 2016-12-04 11:56:02.000000000 +0000 +++ nbformat-4.4.0/debian/patches/0001-Use-setuptools-for-all-targets-so-python-Depends-sub.patch 2017-09-03 14:36:44.000000000 +0000 @@ -1,4 +1,4 @@ -From 1f14738fb23cb2f62ef42e3a73d2a706897886ad Mon Sep 17 00:00:00 2001 +From 50cc84ceb4220c74dcc467e61a7e64f086e624ed Mon Sep 17 00:00:00 2001 From: Julien Puydt Date: Tue, 6 Oct 2015 07:51:30 +0200 Subject: Use setuptools for all targets so python*:Depends substitutions work diff -Nru nbformat-4.2.0/debian/watch nbformat-4.4.0/debian/watch --- nbformat-4.2.0/debian/watch 2016-12-04 11:56:02.000000000 +0000 +++ nbformat-4.4.0/debian/watch 2017-09-03 14:36:44.000000000 +0000 @@ -1,2 +1,2 @@ -version=3 +version=4 https://github.com/jupyter/nbformat/tags .*/archive/(.*)\.tar\.gz diff -Nru nbformat-4.2.0/docs/api.rst nbformat-4.4.0/docs/api.rst --- nbformat-4.2.0/docs/api.rst 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/docs/api.rst 2017-08-18 22:33:37.000000000 +0000 @@ -53,3 +53,70 @@ .. autofunction:: validate .. autoclass:: ValidationError + +Constructing notebooks programmatically +--------------------------------------- + +.. module:: nbformat.v4 + +These functions return :class:`~.NotebookNode` objects with the necessary fields. + +.. autofunction:: new_notebook + +.. autofunction:: new_code_cell + +.. autofunction:: new_markdown_cell + +.. autofunction:: new_raw_cell + +.. autofunction:: new_output + +.. autofunction:: output_from_msg + + +Notebook signatures +------------------- + +.. module:: nbformat.sign + +This machinery is used by the notebook web application to record which notebooks +are *trusted*, and may show dynamic output as soon as they're loaded. See +:ref:`notebook:notebook_security` for more information. + +.. autoclass:: NotebookNotary + + .. automethod:: sign + + .. automethod:: unsign + + .. automethod:: check_signature + + .. automethod:: mark_cells + + .. automethod:: check_cells + +.. _pluggable_signature_store: + +Signature storage +***************** + +Signatures are stored using a pluggable :class:`SignatureStore` subclass. To +implement your own, override the methods below and configure +``NotebookNotary.store_factory``. + +.. autoclass:: SignatureStore + + .. automethod:: store_signature + + .. automethod:: remove_signature + + .. automethod:: check_signature + + .. automethod:: close + +By default, :class:`NotebookNotary` will use an SQLite based store if SQLite +bindings are available, and an in-memory store otherwise. + +.. autoclass:: SQLiteSignatureStore + +.. autoclass:: MemorySignatureStore diff -Nru nbformat-4.2.0/docs/changelog.rst nbformat-4.4.0/docs/changelog.rst --- nbformat-4.2.0/docs/changelog.rst 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/docs/changelog.rst 2017-08-18 22:33:37.000000000 +0000 @@ -4,6 +4,39 @@ Changes in nbformat ========================= +4.4 +=== + +`4.4 on GitHub `__ + +- Explicitly state that metadata fields can be ignored. +- Introduce official jupyter namespace inside metadata (``metadata.jupyter``). +- Introduce ``source_hidden`` and ``outputs_hidden`` as official front-end + metadata fields to indicate hiding source and outputs areas. **NB**: These + fields should not be used to hide elements in exported formats. +- Fix ending the redundant storage of signatures in the signature database. +- :func:`nbformat.validate` can be set to not raise a ValidationError if + additional properties are included. +- Fix for errors with connecting and backing up the signature database. +- Dict-like objects added to NotebookNode attributes are now transformed to be + NotebookNode objects; transformation also works for `.update()`. + + +4.3 +=== + +`4.3 on GitHub `__ + +- A new pluggable ``SignatureStore`` class allows specifying different ways to + record the signatures of trusted notebooks. The default is still an SQLite + database. See :ref:`pluggable_signature_store` for more information. +- :func:`nbformat.read` and :func:`nbformat.write` accept file paths as bytes + as well as unicode. +- Fix for calling :func:`nbformat.validate` on an empty dictionary. +- Fix for running the tests where the locale makes ASCII the default encoding. +- Include nbformat-schema files (v3 and v4) in nbformat-schema npm package. +- Include configuration for appveyor's continuous integration service. + 4.2 === diff -Nru nbformat-4.2.0/docs/conf.py nbformat-4.4.0/docs/conf.py --- nbformat-4.2.0/docs/conf.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/docs/conf.py 2017-08-18 22:33:37.000000000 +0000 @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = '4.0' +version = '4.4' # The full version, including alpha/beta/rc tags. -release = '4.0' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -295,7 +295,8 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'https://docs.python.org/3': None, - 'https://jupyter_client.readthedocs.org/en/stable': None, - 'https://nbconvert.readthedocs.org/en/stable': None, + 'python': ('https://docs.python.org/3', None), + 'jupyterclient': ('https://jupyter_client.readthedocs.org/en/stable', None), + 'nbconvert': ('https://nbconvert.readthedocs.org/en/stable', None), + 'notebook': ('https://jupyter-notebook.readthedocs.org/en/stable', None), } diff -Nru nbformat-4.2.0/docs/format_description.rst nbformat-4.4.0/docs/format_description.rst --- nbformat-4.2.0/docs/format_description.rst 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/docs/format_description.rst 2017-08-18 22:33:37.000000000 +0000 @@ -9,8 +9,9 @@ .. note:: *All* metadata fields are optional. - While the type and values of some metadata are defined, - no metadata values are required to be defined. + While the types and values of some metadata fields are defined, + no metadata fields are required to be defined. Any metadata field + may also be ignored. Top-level structure @@ -312,7 +313,7 @@ { "cell_type" : "markdown", "metadata" : {}, - "source" : ["Here is an *inline* image ![inline image](attachment://test.png)"], + "source" : ["Here is an *inline* image ![inline image](attachment:test.png)"], "attachments" : { "test.png": { "image/png" : "base64-encoded-png-data" @@ -377,6 +378,9 @@ Cell metadata ------------- +Official Jupyter metadata, as used by Jupyter frontends should be placed in the +`metadata.jupyter` namespace, for example `metadata.jupyter.foo = "bar"`. + The following metadata keys are defined at the cell level: =========== =============== ============== @@ -390,6 +394,15 @@ tags list of str A list of string tags on the cell. Commas are not allowed in a tag =========== =============== ============== +The following metadata keys are defined at the cell level within the `jupyter` namespace + +=============== =============== ============== +Key Value Interpretation +=============== =============== ============== +source_hidden bool Whether the cell's source should be shown +outputs_hidden bool Whether the cell's outputs should be shown +=============== =============== ============== + Output metadata --------------- diff -Nru nbformat-4.2.0/index.js nbformat-4.4.0/index.js --- nbformat-4.2.0/index.js 1970-01-01 00:00:00.000000000 +0000 +++ nbformat-4.4.0/index.js 2017-08-18 22:33:37.000000000 +0000 @@ -0,0 +1,3 @@ +exports.v3 = require('./nbformat/v3/nbformat.v3.schema.json'); +exports.v4 = require('./nbformat/v4/nbformat.v4.schema.json'); + diff -Nru nbformat-4.2.0/MANIFEST.in nbformat-4.4.0/MANIFEST.in --- nbformat-4.2.0/MANIFEST.in 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/MANIFEST.in 2017-08-18 22:33:37.000000000 +0000 @@ -2,6 +2,10 @@ include CONTRIBUTING.md include README.md +# Javascript +include package.json +include index.js + # Documentation graft docs exclude docs/\#* diff -Nru nbformat-4.2.0/nbformat/__init__.py nbformat-4.4.0/nbformat/__init__.py --- nbformat-4.2.0/nbformat/__init__.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/__init__.py 2017-08-18 22:33:37.000000000 +0000 @@ -134,7 +134,7 @@ nb : NotebookNode The notebook that was read. """ - if isinstance(fp, py3compat.string_types): + if isinstance(fp, (py3compat.unicode_type, bytes)): with io.open(fp, encoding='utf-8') as f: return read(f, as_version, **kwargs) @@ -159,7 +159,7 @@ If unspecified, or specified as nbformat.NO_CONVERT, the notebook's own version will be used and no conversion performed. """ - if isinstance(fp, py3compat.string_types): + if isinstance(fp, (py3compat.unicode_type, bytes)): with io.open(fp, 'w', encoding='utf-8') as f: return write(nb, f, version=version, **kwargs) diff -Nru nbformat-4.2.0/nbformat/notebooknode.py nbformat-4.4.0/nbformat/notebooknode.py --- nbformat-4.2.0/nbformat/notebooknode.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/notebooknode.py 2017-08-18 22:33:37.000000000 +0000 @@ -1,23 +1,50 @@ """NotebookNode - adding attribute access to dicts""" from ipython_genutils.ipstruct import Struct +from collections import Mapping + class NotebookNode(Struct): """A dict-like node with attribute-access""" - pass + + def __setitem__(self, key, value): + if isinstance(value, Mapping) and not isinstance(value, NotebookNode): + value = from_dict(value) + super(NotebookNode, self).__setitem__(key, value) + + def update(self, *args, **kwargs): + """ + A dict-like update method based on CPython's MutableMapping `update` + method. + """ + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwargs.items(): + self[key] = value + def from_dict(d): """Convert dict to dict-like NotebookNode - + Recursively converts any dict in the container to a NotebookNode. This does not check that the contents of the dictionary make a valid notebook or part of a notebook. """ if isinstance(d, dict): - return NotebookNode({k:from_dict(v) for k,v in d.items()}) + return NotebookNode({k: from_dict(v) for k, v in d.items()}) elif isinstance(d, (tuple, list)): return [from_dict(i) for i in d] else: return d - - diff -Nru nbformat-4.2.0/nbformat/sign.py nbformat-4.4.0/nbformat/sign.py --- nbformat-4.2.0/nbformat/sign.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/sign.py 2017-08-18 22:33:37.000000000 +0000 @@ -4,6 +4,7 @@ # Distributed under the terms of the Modified BSD License. import base64 +from collections import OrderedDict from contextlib import contextmanager from datetime import datetime import hashlib @@ -22,7 +23,7 @@ from ipython_genutils.py3compat import unicode_type, cast_bytes, cast_unicode from traitlets import ( - Instance, Bytes, Enum, Any, Unicode, Bool, Integer, + Instance, Bytes, Enum, Any, Unicode, Bool, Integer, TraitType, default, observe, ) from traitlets.config import LoggingConfigurable, MultipleInstanceError @@ -40,6 +41,224 @@ algorithms = hashlib.algorithms +# This has been added to traitlets, but is not released as of traitlets 4.3.1, +# so a copy is included here for now. +class Callable(TraitType): + """A trait which is callable. + + Notes + ----- + Classes are callable, as are instances + with a __call__() method.""" + + info_text = 'a callable' + + def validate(self, obj, value): + if callable(value): + return value + else: + self.error(obj, value) + + +class SignatureStore(object): + """Base class for a signature store.""" + def store_signature(self, digest, algorithm): + """Implement in subclass to store a signature. + + Should not raise if the signature is already stored. + """ + raise NotImplementedError + + def check_signature(self, digest, algorithm): + """Implement in subclass to check if a signature is known. + + Return True for a known signature, False for unknown. + """ + raise NotImplementedError + + def remove_signature(self, digest, algorithm): + """Implement in subclass to delete a signature. + + Should not raise if the signature is not stored. + """ + raise NotImplementedError + + def close(self): + """Close any open connections this store may use. + + If the store maintains any open connections (e.g. to a database), + they should be closed. + """ + pass + + +class MemorySignatureStore(SignatureStore): + """Non-persistent storage of signatures in memory. + """ + cache_size = 65535 + def __init__(self): + # We really only want an ordered set, but the stdlib has OrderedDict, + # and it's easy to use a dict as a set. + self.data = OrderedDict() + + def store_signature(self, digest, algorithm): + key = (digest, algorithm) + # Pop it so it goes to the end when we reinsert it + self.data.pop(key, None) + self.data[key] = None + + self._maybe_cull() + + def _maybe_cull(self): + """If more than cache_size signatures are stored, delete the oldest 25% + """ + if len(self.data) < self.cache_size: + return + + for _ in range(len(self.data) // 4): + self.data.popitem(last=False) + + def check_signature(self, digest, algorithm): + key = (digest, algorithm) + if key in self.data: + # Move it to the end (.move_to_end() method is new in Py3) + del self.data[key] + self.data[key] = None + return True + return False + + def remove_signature(self, digest, algorithm): + self.data.pop((digest, algorithm), None) + +class SQLiteSignatureStore(SignatureStore, LoggingConfigurable): + """Store signatures in an SQLite database. + """ + # 64k entries ~ 12MB + cache_size = Integer(65535, + help="""The number of notebook signatures to cache. + When the number of signatures exceeds this value, + the oldest 25% of signatures will be culled. + """ + ).tag(config=True) + + def __init__(self, db_file, **kwargs): + super(SQLiteSignatureStore, self).__init__(**kwargs) + self.db_file = db_file + self.db = self._connect_db(db_file) + + def close(self): + if self.db is not None: + self.db.close() + + def _connect_db(self, db_file): + kwargs = dict( + detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) + db = None + try: + db = sqlite3.connect(db_file, **kwargs) + self.init_db(db) + except (sqlite3.DatabaseError, sqlite3.OperationalError): + if db_file != ':memory:': + old_db_location = db_file + ".bak" + if db is not None: + db.close() + self.log.warning( + ("The signatures database cannot be opened; maybe it is corrupted or encrypted. " + "You may need to rerun your notebooks to ensure that they are trusted to run Javascript. " + "The old signatures database has been renamed to %s and a new one has been created."), + old_db_location) + try: + os.rename(db_file, old_db_location) + db = sqlite3.connect(db_file, **kwargs) + self.init_db(db) + except (sqlite3.DatabaseError, sqlite3.OperationalError, OSError): + if db is not None: + db.close() + self.log.warning( + ("Failed commiting signatures database to disk. " + "You may need to move the database file to a non-networked file system, " + "using config option `NotebookNotary.db_file`. " + "Using in-memory signatures database for the remainder of this session.")) + self.db_file = ':memory:' + db = sqlite3.connect(':memory:', **kwargs) + self.init_db(db) + else: + raise + return db + + def init_db(self, db): + db.execute(""" + CREATE TABLE IF NOT EXISTS nbsignatures + ( + id integer PRIMARY KEY AUTOINCREMENT, + algorithm text, + signature text, + path text, + last_seen timestamp + )""") + db.execute(""" + CREATE INDEX IF NOT EXISTS algosig ON nbsignatures(algorithm, signature) + """) + db.commit() + + def store_signature(self, digest, algorithm): + if self.db is None: + return + if not self.check_signature(digest, algorithm): + self.db.execute(""" + INSERT INTO nbsignatures (algorithm, signature, last_seen) + VALUES (?, ?, ?) + """, (algorithm, digest, datetime.utcnow()) + ) + else: + self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE + algorithm = ? AND + signature = ?; + """, (datetime.utcnow(), algorithm, digest) + ) + self.db.commit() + + # Check size and cull old entries if necessary + n, = self.db.execute("SELECT Count(*) FROM nbsignatures").fetchone() + if n > self.cache_size: + self.cull_db() + + def check_signature(self, digest, algorithm): + if self.db is None: + return False + r = self.db.execute("""SELECT id FROM nbsignatures WHERE + algorithm = ? AND + signature = ?; + """, (algorithm, digest)).fetchone() + if r is None: + return False + self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE + algorithm = ? AND + signature = ?; + """, + (datetime.utcnow(), algorithm, digest), + ) + self.db.commit() + return True + + def remove_signature(self, digest, algorithm): + self.db.execute("""DELETE FROM nbsignatures WHERE + algorithm = ? AND + signature = ?; + """, + (algorithm, digest) + ) + + self.db.commit() + + def cull_db(self): + """Cull oldest 25% of the trusted signatures when the size limit is reached""" + self.db.execute("""DELETE FROM nbsignatures WHERE id IN ( + SELECT id FROM nbsignatures ORDER BY last_seen DESC LIMIT -1 OFFSET ? + ); + """, (max(int(0.75 * self.cache_size), 1),)) + + def yield_everything(obj): """Yield every item in a container as bytes @@ -107,7 +326,20 @@ app = JupyterApp() app.initialize(argv=[]) return app.data_dir - + + store_factory = Callable( + help="""A callable returning the storage backend for notebook signatures. + The default uses an SQLite database.""").tag(config=True) + + @default('store_factory') + def _store_factory_default(self): + def factory(): + if sqlite3 is None: + self.log.warning("Missing SQLite3, all notebooks will be untrusted!") + return MemorySignatureStore() + return SQLiteSignatureStore(self.db_file) + return factory + db_file = Unicode( help="""The sqlite file in which to store notebook signatures. By default, this will be in your Jupyter data directory. @@ -120,56 +352,6 @@ return ':memory:' return os.path.join(self.data_dir, u'nbsignatures.db') - # 64k entries ~ 12MB - cache_size = Integer(65535, - help="""The number of notebook signatures to cache. - When the number of signatures exceeds this value, - the oldest 25% of signatures will be culled. - """ - ).tag(config=True) - db = Any() - @default('db') - def _db_default(self): - if sqlite3 is None: - self.log.warn("Missing SQLite3, all notebooks will be untrusted!") - return - kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) - try: - db = sqlite3.connect(self.db_file, **kwargs) - self.init_db(db) - except (sqlite3.DatabaseError, sqlite3.OperationalError): - if self.db_file != ':memory:': - old_db_location = os.path.join(self.data_dir, self.db_file + ".bak") - self.log.warn("""The signatures database cannot be opened; maybe it is corrupted or encrypted. You may need to rerun your notebooks to ensure that they are trusted to run Javascript. The old signatures database has been renamed to %s and a new one has been created.""", - old_db_location) - try: - os.rename(self.db_file, self.db_file + u'.bak') - db = sqlite3.connect(self.db_file, **kwargs) - self.init_db(db) - except (sqlite3.DatabaseError, sqlite3.OperationalError): - self.log.warn("""Failed commiting signatures database to disk. You may need to move the database file to a non-networked file system, using config option `NotebookNotary.db_file`. Using in-memory signatures database for the remainder of this session.""") - self.db_file = ':memory:' - db = sqlite3.connect(self.db_file, **kwargs) - self.init_db(db) - else: - raise - return db - - def init_db(self, db): - db.execute(""" - CREATE TABLE IF NOT EXISTS nbsignatures - ( - id integer PRIMARY KEY AUTOINCREMENT, - algorithm text, - signature text, - path text, - last_seen timestamp - )""") - db.execute(""" - CREATE INDEX IF NOT EXISTS algosig ON nbsignatures(algorithm, signature) - """) - db.commit() - algorithm = Enum(algorithms, default_value='sha256', help="""The hashing algorithm used to sign notebooks.""" ).tag(config=True) @@ -204,6 +386,10 @@ secret = base64.encodestring(os.urandom(1024)) self._write_secret_file(secret) return secret + + def __init__(self, **kwargs): + super(NotebookNotary, self).__init__(**kwargs) + self.store = self.store_factory() def _write_secret_file(self, secret): """write my secret to my secret_file""" @@ -213,7 +399,7 @@ try: os.chmod(self.secret_file, 0o600) except OSError: - self.log.warn( + self.log.warning( "Could not set permissions on %s", self.secret_file ) @@ -249,23 +435,8 @@ """ if nb.nbformat < 3: return False - if self.db is None: - return False signature = self.compute_signature(nb) - r = self.db.execute("""SELECT id FROM nbsignatures WHERE - algorithm = ? AND - signature = ?; - """, (self.algorithm, signature)).fetchone() - if r is None: - return False - self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE - algorithm = ? AND - signature = ?; - """, - (datetime.utcnow(), self.algorithm, signature), - ) - self.db.commit() - return True + return self.store.check_signature(signature, self.algorithm) def sign(self, nb): """Sign a notebook, indicating that its output is trusted on this machine @@ -275,25 +446,7 @@ if nb.nbformat < 3: return signature = self.compute_signature(nb) - self.store_signature(signature, nb) - - def store_signature(self, signature, nb): - if self.db is None: - return - self.db.execute("""INSERT OR IGNORE INTO nbsignatures - (algorithm, signature, last_seen) VALUES (?, ?, ?)""", - (self.algorithm, signature, datetime.utcnow()) - ) - self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE - algorithm = ? AND - signature = ?; - """, - (datetime.utcnow(), self.algorithm, signature), - ) - self.db.commit() - n, = self.db.execute("SELECT Count(*) FROM nbsignatures").fetchone() - if n > self.cache_size: - self.cull_db() + self.store.store_signature(signature, self.algorithm) def unsign(self, nb): """Ensure that a notebook is untrusted @@ -301,26 +454,14 @@ by removing its signature from the trusted database, if present. """ signature = self.compute_signature(nb) - self.db.execute("""DELETE FROM nbsignatures WHERE - algorithm = ? AND - signature = ?; - """, - (self.algorithm, signature) - ) - self.db.commit() - - def cull_db(self): - """Cull oldest 25% of the trusted signatures when the size limit is reached""" - self.db.execute("""DELETE FROM nbsignatures WHERE id IN ( - SELECT id FROM nbsignatures ORDER BY last_seen DESC LIMIT -1 OFFSET ? - ); - """, (max(int(0.75 * self.cache_size), 1),)) + self.store.remove_signature(signature, self.algorithm) def mark_cells(self, nb, trusted): """Mark cells as trusted if the notebook's signature can be verified Sets ``cell.metadata.trusted = True | False`` on all code cells, - depending on whether the stored signature can be verified. + depending on the *trusted* parameter. This will typically be the return + value from ``self.check_signature(nb)``. This function is the inverse of check_cells """ @@ -364,8 +505,10 @@ return True def check_cells(self, nb): - """Return whether all code cells are trusted + """Return whether all code cells are trusted. + A cell is trusted if the 'trusted' field in its metadata is truthy, or + if it has no potentially unsafe outputs. If there are no code cells, return True. This function is the inverse of mark_cells. @@ -399,6 +542,10 @@ Otherwise, you will have to re-execute the notebook to see output. """ + # This command line tool should use the same config file as the notebook + @default('config_file_name') + def _config_file_name_default(self): + return 'jupyter_notebook_config' examples = """ jupyter trust mynotebook.ipynb and_this_one.ipynb diff -Nru nbformat-4.2.0/nbformat/tests/base.py nbformat-4.4.0/nbformat/tests/base.py --- nbformat-4.2.0/nbformat/tests/base.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/tests/base.py 2017-08-18 22:33:37.000000000 +0000 @@ -7,13 +7,13 @@ import os import unittest - +import io class TestsBase(unittest.TestCase): """Base tests class.""" - def fopen(self, f, mode=u'r'): - return open(os.path.join(self._get_files_path(), f), mode) + def fopen(self, f, mode=u'r',encoding='utf-8'): + return io.open(os.path.join(self._get_files_path(), f), mode, encoding=encoding) def _get_files_path(self): diff -Nru nbformat-4.2.0/nbformat/tests/test4jupyter_metadata.ipynb nbformat-4.4.0/nbformat/tests/test4jupyter_metadata.ipynb --- nbformat-4.2.0/nbformat/tests/test4jupyter_metadata.ipynb 1970-01-01 00:00:00.000000000 +0000 +++ nbformat-4.4.0/nbformat/tests/test4jupyter_metadata.ipynb 2017-08-18 22:33:37.000000000 +0000 @@ -0,0 +1,30 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello\n" + ] + } + ], + "source": [ + "print(\"hello\")" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 0 +} diff -Nru nbformat-4.2.0/nbformat/tests/test_sign.py nbformat-4.4.0/nbformat/tests/test_sign.py --- nbformat-4.2.0/nbformat/tests/test_sign.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/tests/test_sign.py 2017-08-18 22:33:37.000000000 +0000 @@ -12,9 +12,11 @@ import time import tempfile import testpath +import unittest from .base import TestsBase +from traitlets.config import Config from nbformat import read, sign, write class TestNotary(TestsBase): @@ -32,6 +34,7 @@ self.nb3 = read(f, as_version=3) def tearDown(self): + self.notary.store.close() shutil.rmtree(self.data_dir) def test_invalid_db_file(self): @@ -44,9 +47,11 @@ secret=b'secret', ) invalid_notary.sign(self.nb) + invalid_notary.store.close() testpath.assert_isfile(os.path.join(self.data_dir, invalid_sql_file)) testpath.assert_isfile(os.path.join(self.data_dir, invalid_sql_file + '.bak')) + def test_algorithms(self): last_sig = '' @@ -89,9 +94,9 @@ nbs = [ copy.deepcopy(self.nb) for i in range(10) ] - for row in self.notary.db.execute("SELECT * FROM nbsignatures"): + for row in self.notary.store.db.execute("SELECT * FROM nbsignatures"): print(row) - self.notary.cache_size = 8 + self.notary.store.cache_size = 8 for i, nb in enumerate(nbs[:8]): nb.metadata.dirty = i self.notary.sign(nb) @@ -222,9 +227,36 @@ p.stdin.close() p.wait() self.assertEqual(p.returncode, 0) - return p.stdout.read().decode('utf8', 'replace') + out = p.stdout.read().decode('utf8', 'replace') + p.stdout.close() + return out out = sign_stdin(self.nb3) self.assertIn('Signing notebook: ', out) out = sign_stdin(self.nb3) self.assertIn('already signed: ', out) + +def test_config_store(): + store = sign.MemorySignatureStore() + + c = Config() + c.NotebookNotary.store_factory = lambda: store + notary = sign.NotebookNotary(config=c) + assert notary.store is store + +class SignatureStoreTests(unittest.TestCase): + def setUp(self): + self.store = sign.MemorySignatureStore() + + def test_basics(self): + digest = '0123457689abcef' + algo = 'fake_sha' + assert not self.store.check_signature(digest, algo) + self.store.store_signature(digest, algo) + assert self.store.check_signature(digest, algo) + self.store.remove_signature(digest, algo) + assert not self.store.check_signature(digest, algo) + +class SQLiteSignatureStoreTests(SignatureStoreTests): + def setUp(self): + self.store = sign.SQLiteSignatureStore(':memory:') diff -Nru nbformat-4.2.0/nbformat/tests/test_validator.py nbformat-4.4.0/nbformat/tests/test_validator.py --- nbformat-4.2.0/nbformat/tests/test_validator.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/tests/test_validator.py 2017-08-18 22:33:37.000000000 +0000 @@ -48,6 +48,13 @@ validate(nb) self.assertEqual(isvalid(nb), True) + def test_nb4jupyter_metadata(self): + """Test that a notebook with a jupyter metadata passes validation""" + with self.fopen(u'test4jupyter_metadata.ipynb', u'r') as f: + nb = read(f, as_version=4) + validate(nb) + self.assertEqual(isvalid(nb), True) + def test_invalid(self): """Test than an invalid notebook does not pass validation""" # this notebook has a few different errors: @@ -60,6 +67,10 @@ validate(nb) self.assertEqual(isvalid(nb), False) + def test_validate_empty(self): + """Test that an empty dict can be validated without error""" + validate({}) + def test_future(self): """Test than a notebook from the future with extra keys passes validation""" with self.fopen(u'test4plus.ipynb', u'r') as f: diff -Nru nbformat-4.2.0/nbformat/v4/__init__.py nbformat-4.4.0/nbformat/v4/__init__.py --- nbformat-4.2.0/nbformat/v4/__init__.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/v4/__init__.py 2017-08-18 22:33:37.000000000 +0000 @@ -9,7 +9,7 @@ from .nbbase import ( nbformat, nbformat_minor, nbformat_schema, - new_code_cell, new_markdown_cell, new_notebook, + new_code_cell, new_markdown_cell, new_raw_cell, new_notebook, new_output, output_from_msg, ) diff -Nru nbformat-4.2.0/nbformat/v4/nbbase.py nbformat-4.4.0/nbformat/v4/nbbase.py --- nbformat-4.2.0/nbformat/v4/nbbase.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/v4/nbbase.py 2017-08-18 22:33:37.000000000 +0000 @@ -9,7 +9,7 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from ..notebooknode import from_dict, NotebookNode +from ..notebooknode import NotebookNode # Change this when incrementing the nbformat version nbformat = 4 @@ -35,9 +35,9 @@ output.metadata = NotebookNode() output.data = NotebookNode() # load from args: - output.update(from_dict(kwargs)) + output.update(kwargs) if data is not None: - output.data = from_dict(data) + output.data = data # validate validate(output, output_type) return output @@ -95,7 +95,7 @@ source=source, outputs=[], ) - cell.update(from_dict(kwargs)) + cell.update(kwargs) validate(cell, 'code_cell') return cell @@ -107,7 +107,7 @@ source=source, metadata=NotebookNode(), ) - cell.update(from_dict(kwargs)) + cell.update(kwargs) validate(cell, 'markdown_cell') return cell @@ -119,7 +119,7 @@ source=source, metadata=NotebookNode(), ) - cell.update(from_dict(kwargs)) + cell.update(kwargs) validate(cell, 'raw_cell') return cell @@ -132,6 +132,6 @@ metadata=NotebookNode(), cells=[], ) - nb.update(from_dict(kwargs)) + nb.update(kwargs) validate(nb) return nb diff -Nru nbformat-4.2.0/nbformat/v4/nbformat.v4.schema.json nbformat-4.4.0/nbformat/v4/nbformat.v4.schema.json --- nbformat-4.2.0/nbformat/v4/nbformat.v4.schema.json 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/v4/nbformat.v4.schema.json 2017-08-18 22:33:37.000000000 +0000 @@ -126,6 +126,15 @@ "description": "Raw cell metadata format for nbconvert.", "type": "string" }, + "jupyter": { + "description": "Official Jupyter Metadata for Raw Cells", + "type": "object", + "additionalProperties": true, + "source_hidden": { + "description": "Whether the source is hidden.", + "type": "boolean" + } + }, "name": {"$ref": "#/definitions/misc/metadata_name"}, "tags": {"$ref": "#/definitions/misc/metadata_tags"} } @@ -150,7 +159,16 @@ "type": "object", "properties": { "name": {"$ref": "#/definitions/misc/metadata_name"}, - "tags": {"$ref": "#/definitions/misc/metadata_tags"} + "tags": {"$ref": "#/definitions/misc/metadata_tags"}, + "jupyter": { + "description": "Official Jupyter Metadata for Markdown Cells", + "type": "object", + "additionalProperties": true, + "source_hidden": { + "description": "Whether the source is hidden.", + "type": "boolean" + } + } }, "additionalProperties": true }, @@ -174,6 +192,19 @@ "type": "object", "additionalProperties": true, "properties": { + "jupyter": { + "description": "Official Jupyter Metadata for Code Cells", + "type": "object", + "additionalProperties": true, + "source_hidden": { + "description": "Whether the source is hidden.", + "type": "boolean" + }, + "outputs_hidden": { + "description": "Whether the outputs are hidden.", + "type": "boolean" + } + }, "collapsed": { "description": "Whether the cell is collapsed/expanded.", "type": "boolean" diff -Nru nbformat-4.2.0/nbformat/v4/nbjson.py nbformat-4.4.0/nbformat/v4/nbjson.py --- nbformat-4.2.0/nbformat/v4/nbjson.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/v4/nbjson.py 2017-08-18 22:33:37.000000000 +0000 @@ -8,7 +8,7 @@ from ipython_genutils import py3compat -from .nbbase import from_dict +from ..notebooknode import from_dict from .rwbase import ( NotebookReader, NotebookWriter, rejoin_lines, split_lines, strip_transient ) diff -Nru nbformat-4.2.0/nbformat/v4/rwbase.py nbformat-4.4.0/nbformat/v4/rwbase.py --- nbformat-4.2.0/nbformat/v4/rwbase.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/v4/rwbase.py 2017-08-18 22:33:37.000000000 +0000 @@ -106,7 +106,7 @@ def reads(self, s, **kwargs): """Read a notebook from a string.""" - raise NotImplementedError("loads must be implemented in a subclass") + raise NotImplementedError("reads must be implemented in a subclass") def read(self, fp, **kwargs): """Read a notebook from a file like object""" @@ -119,7 +119,7 @@ def writes(self, nb, **kwargs): """Write a notebook to a string.""" - raise NotImplementedError("loads must be implemented in a subclass") + raise NotImplementedError("writes must be implemented in a subclass") def write(self, nb, fp, **kwargs): """Write a notebook to a file like object""" diff -Nru nbformat-4.2.0/nbformat/validator.py nbformat-4.4.0/nbformat/validator.py --- nbformat-4.2.0/nbformat/validator.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/validator.py 2017-08-18 22:33:37.000000000 +0000 @@ -15,9 +15,9 @@ verbose_msg = """ Jupyter notebook format depends on the jsonschema package: - + https://pypi.python.org/pypi/jsonschema - + Please install it first. """ raise ImportError(str(e) + verbose_msg) @@ -50,14 +50,14 @@ ) return schema -def get_validator(version=None, version_minor=None): +def get_validator(version=None, version_minor=None, relax_add_props=False): """Load the JSON schema into a Validator""" if version is None: from .. import current_nbformat version = current_nbformat v = import_item("nbformat.v%s" % version) - current_minor = v.nbformat_minor + current_minor = getattr(v, 'nbformat_minor', 0) if version_minor is None: version_minor = current_minor @@ -65,13 +65,9 @@ if version_tuple not in validators: try: - v.nbformat_schema + schema_json = _get_schema_json(v) except AttributeError: - # no validator return None - schema_path = os.path.join(os.path.dirname(v.__file__), v.nbformat_schema) - with open(schema_path) as f: - schema_json = json.load(f) if current_minor < version_minor: # notebook from the future, relax all `additionalProperties: False` requirements @@ -80,8 +76,30 @@ schema_json = _allow_undefined(schema_json) validators[version_tuple] = Validator(schema_json) + + if relax_add_props: + try: + schema_json = _get_schema_json(v) + except AttributeError: + return None + + # this allows properties to be added for intermediate + # representations while validating for all other kinds of errors + schema_json = _relax_additional_properties(schema_json) + + validators[version_tuple] = Validator(schema_json) return validators[version_tuple] + +def _get_schema_json(v): + """ + Gets the json schema from a given imported library a nbformat version. + """ + schema_path = os.path.join(os.path.dirname(v.__file__), v.nbformat_schema) + with open(schema_path) as f: + schema_json = json.load(f) + return schema_json + def isvalid(nbjson, ref=None, version=None, version_minor=None): """Checks whether the given notebook JSON conforms to the current notebook format schema. Returns True if the JSON is valid, and @@ -100,7 +118,7 @@ def _format_as_index(indices): """ (from jsonschema._utils.format_as_index, copied to avoid relying on private API) - + Construct a single string containing indexing operations for the indices. For example, [1, 2, "foo"] -> [1][2]["foo"] @@ -115,7 +133,7 @@ def _truncate_obj(obj): """Truncate objects for use in validation tracebacks - + Cell and output lists are squashed, as are long strings, lists, and dicts. """ if isinstance(obj, dict): @@ -143,25 +161,25 @@ class NotebookValidationError(ValidationError): """Schema ValidationError with truncated representation - + to avoid massive verbose tracebacks. """ def __init__(self, original, ref=None): self.original = original self.ref = getattr(self.original, 'ref', ref) self.message = self.original.message - + def __getattr__(self, key): return getattr(self.original, key) - + def __unicode__(self): """Custom str for validation errors - + avoids dumping full schema and notebook to logs """ error = self.original instance = _truncate_obj(error.instance) - + return u'\n'.join([ error.message, u'', @@ -173,7 +191,7 @@ u'On instance%s:' % _format_as_index(error.relative_path), pprint.pformat(instance, width=78), ]) - + if sys.version_info >= (3,): __str__ = __unicode__ @@ -216,7 +234,7 @@ return NotebookValidationError(error, ref) -def validate(nbjson, ref=None, version=None, version_minor=None): +def validate(nbjson, ref=None, version=None, version_minor=None, relax_add_props=False): """Checks whether the given notebook JSON conforms to the current notebook format schema. @@ -226,7 +244,7 @@ from .reader import get_version (version, version_minor) = get_version(nbjson) - validator = get_validator(version, version_minor) + validator = get_validator(version, version_minor, relax_add_props=relax_add_props) if validator is None: # no validator diff -Nru nbformat-4.2.0/nbformat/_version.py nbformat-4.4.0/nbformat/_version.py --- nbformat-4.2.0/nbformat/_version.py 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/nbformat/_version.py 2017-08-18 22:33:37.000000000 +0000 @@ -1,2 +1,4 @@ -version_info = (4, 2, 0) +# Make sure to update package.json, too! +# version_info = (4, 5, 0, 'dev') +version_info = (4, 4, 0) __version__ = '.'.join(map(str, version_info)) diff -Nru nbformat-4.2.0/package.json nbformat-4.4.0/package.json --- nbformat-4.2.0/package.json 1970-01-01 00:00:00.000000000 +0000 +++ nbformat-4.4.0/package.json 2017-08-18 22:33:37.000000000 +0000 @@ -0,0 +1,25 @@ +{ + "name": "nbformat-schema", + "version": "4.4.0", + "description": "JSON schemata for Jupyter notebook formats", + "main": "index.js", + "files": [ + "nbformat/v3/nbformat.v3.schema.json", + "nbformat/v4/nbformat.v4.schema.json" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/jupyter/nbformat.git" + }, + "keywords": [ + "jupyter", + "notebook", + "json-schema" + ], + "author": "Project Jupyter Contributors", + "license": "BSD-3-Clause", + "bugs": { + "url": "https://github.com/jupyter/nbformat/issues" + }, + "homepage": "https://nbformat.readthedocs.io" +} diff -Nru nbformat-4.2.0/.travis.yml nbformat-4.4.0/.travis.yml --- nbformat-4.2.0/.travis.yml 2016-12-01 10:51:43.000000000 +0000 +++ nbformat-4.4.0/.travis.yml 2017-08-18 22:33:37.000000000 +0000 @@ -1,6 +1,7 @@ language: python python: - nightly + - 3.6 - 3.5 - 3.4 - 3.3