diff -Nru python-ijson-3.1.3/CHANGELOG.md python-ijson-3.1.4/CHANGELOG.md --- python-ijson-3.1.3/CHANGELOG.md 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/CHANGELOG.md 2021-03-02 09:59:40.000000000 +0000 @@ -1,5 +1,14 @@ # Changelog +## [3.1.4] + +* Fixed bug in ``yajl2_c`` backend introduced in 3.1.0 + where ``ijson.items`` didn't work correctly + against member names containing ``.`` (#41). +* Python backend raises errors on incomplete JSON content + that previously wasn't recognised as such, + aligning itself with the rest of the backends (#42). + ## [3.1.3] * Python backed correctly raises errors @@ -269,3 +278,4 @@ [3.1.2]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2 [3.1.2.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2.post0 [3.1.3]: https://github.com/ICRAR/ijson/releases/tag/v3.1.3 +[3.1.4]: https://github.com/ICRAR/ijson/releases/tag/v3.1.4 diff -Nru python-ijson-3.1.3/debian/changelog python-ijson-3.1.4/debian/changelog --- python-ijson-3.1.3/debian/changelog 2021-01-30 19:17:54.000000000 +0000 +++ python-ijson-3.1.4/debian/changelog 2021-03-06 12:21:35.000000000 +0000 @@ -1,3 +1,9 @@ +python-ijson (3.1.4-1) unstable; urgency=medium + + * New upstream version 3.1.4 + + -- Tomasz Buchert Sat, 06 Mar 2021 13:21:35 +0100 + python-ijson (3.1.3-1) unstable; urgency=medium [ Debian Janitor ] diff -Nru python-ijson-3.1.3/ijson/backends/python.py python-ijson-3.1.4/ijson/backends/python.py --- python-ijson-3.1.3/ijson/backends/python.py 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/python.py 2021-03-02 09:59:40.000000000 +0000 @@ -226,6 +226,8 @@ if number == inf: raise common.JSONError("float overflow: %s" % (symbol,)) except: + if 'true'.startswith(symbol) or 'false'.startswith(symbol) or 'null'.startswith(symbol): + raise common.IncompleteJSONError('Incomplete JSON content') raise UnexpectedSymbol(symbol, pos) else: send(('number', number)) diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/items_async.c python-ijson-3.1.4/ijson/backends/yajl2_c/items_async.c --- python-ijson-3.1.3/ijson/backends/yajl2_c/items_async.c 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/items_async.c 2021-03-02 09:59:40.000000000 +0000 @@ -32,17 +32,15 @@ { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); - PyObject *parse_args = PyTuple_Pack(1, Py_False); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, - {&ParseBasecoro_Type, parse_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); Py_DECREF(items_args); - Py_DECREF(parse_args); Py_DECREF(reading_args); return 0; } diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/items.c python-ijson-3.1.4/ijson/backends/yajl2_c/items.c --- python-ijson-3.1.3/ijson/backends/yajl2_c/items.c 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/items.c 2021-03-02 09:59:40.000000000 +0000 @@ -21,16 +21,14 @@ { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *items_args = PySequence_GetSlice(args, 2, 4); - PyObject *parse_args = PyTuple_Pack(1, Py_False); pipeline_node coro_pipeline[] = { {&ItemsBasecoro_Type, items_args, NULL}, - {&ParseBasecoro_Type, parse_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, reading_args, coro_pipeline)); Py_DECREF(items_args); - Py_DECREF(parse_args); Py_DECREF(reading_args); return 0; } diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/kvitems_async.c python-ijson-3.1.4/ijson/backends/yajl2_c/kvitems_async.c --- python-ijson-3.1.3/ijson/backends/yajl2_c/kvitems_async.c 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/kvitems_async.c 2021-03-02 09:59:40.000000000 +0000 @@ -32,16 +32,14 @@ { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); - PyObject *parse_args = PyTuple_Pack(1, Py_False); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, - {&ParseBasecoro_Type, parse_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); async_reading_generator_add_coro(self->reading_generator, coro_pipeline); - Py_DECREF(parse_args); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return 0; diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/kvitems.c python-ijson-3.1.4/ijson/backends/yajl2_c/kvitems.c --- python-ijson-3.1.3/ijson/backends/yajl2_c/kvitems.c 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/kvitems.c 2021-03-02 09:59:40.000000000 +0000 @@ -21,15 +21,13 @@ { PyObject *reading_args = PySequence_GetSlice(args, 0, 2); PyObject *kvitems_args = PySequence_GetSlice(args, 2, 4); - PyObject *parse_args = PyTuple_Pack(1, Py_False); pipeline_node coro_pipeline[] = { {&KVItemsBasecoro_Type, kvitems_args, NULL}, - {&ParseBasecoro_Type, parse_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, {&BasicParseBasecoro_Type, NULL, kwargs}, {NULL} }; M1_M1(reading_generator_init(&self->reading_gen, reading_args, coro_pipeline)); - Py_DECREF(parse_args); Py_DECREF(kvitems_args); Py_DECREF(reading_args); return 0; diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/parse_basecoro.c python-ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.c --- python-ijson-3.1.3/ijson/backends/yajl2_c/parse_basecoro.c 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.c 2021-03-02 09:59:40.000000000 +0000 @@ -19,21 +19,16 @@ */ static int parse_basecoro_init(ParseBasecoro *self, PyObject *args, PyObject *kwargs) { - PyObject *join_path = Py_True; - M1_Z(PyArg_ParseTuple(args, "O|O", &self->target_send, &join_path)); + M1_Z(PyArg_ParseTuple(args, "O", &self->target_send)); Py_INCREF(self->target_send); M1_N(self->path = PyList_New(0)); - self->join_path = PyObject_IsTrue(join_path); - assert(self->join_path || - (KVItemsBasecoro_Check(self->target_send) || ItemsBasecoro_Check(self->target_send))); - - if (self->join_path) { - PyObject *empty; - M1_N(empty = STRING_FROM_UTF8("", 0)); - int res = PyList_Append(self->path, empty); - Py_DECREF(empty); - M1_M1(res); - } + + PyObject *empty; + M1_N(empty = STRING_FROM_UTF8("", 0)); + int res = PyList_Append(self->path, empty); + Py_DECREF(empty); + M1_M1(res); + return 0; } @@ -51,8 +46,9 @@ N_N(tgt); \ } while(0); -static PyObject *_send_impl_joint_path(ParseBasecoro *gen, PyObject *event, PyObject *value) +PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value) { + ParseBasecoro *gen = (ParseBasecoro *)self; Py_ssize_t npaths = PyList_Size(gen->path); // Calculate current prefix @@ -110,60 +106,21 @@ N_M1(PyList_Append(gen->path, Py_None)); } - PyObject *res = PyTuple_Pack(3, prefix, event, value); - CORO_SEND(gen->target_send, res); - Py_DECREF(res); - Py_DECREF(prefix); - Py_RETURN_NONE; -} - -static PyObject *_to_next_coro(ParseBasecoro *gen, PyObject *path, PyObject *event, PyObject *value) -{ if (KVItemsBasecoro_Check(gen->target_send)) { - return kvitems_basecoro_send_impl(gen->target_send, path, event, value); + kvitems_basecoro_send_impl(gen->target_send, prefix, event, value); } - return items_basecoro_send_impl(gen->target_send, path, event, value); -} - -static PyObject *_send_impl(ParseBasecoro *gen, PyObject *event, PyObject *value) -{ - if (event == enames.end_array_ename || event == enames.end_map_ename) { - Py_ssize_t npaths = PyList_Size(gen->path); - N_M1(PyList_SetSlice(gen->path, npaths - 1, npaths, NULL)); - N_N(_to_next_coro(gen, gen->path, event, value)); - } - else if (event == enames.start_array_ename) { - N_N(_to_next_coro(gen, gen->path, event, value)); - N_M1(PyList_Append(gen->path, item)); - } - else if (event == enames.start_map_ename) { - N_N(_to_next_coro(gen, gen->path, event, value)); - Py_INCREF(Py_None); - N_M1(PyList_Append(gen->path, Py_None)); - } - else if (event == enames.map_key_ename) { - Py_ssize_t npaths = PyList_Size(gen->path); - PyObject *path = PyList_GetSlice(gen->path, 0, npaths - 1); - N_N(_to_next_coro(gen, path, event, value)); - Py_DECREF(path); - Py_INCREF(value); - N_M1(PyList_SetItem(gen->path, npaths - 1, value)); + else if (ItemsBasecoro_Check(gen->target_send)) { + items_basecoro_send_impl(gen->target_send, prefix, event, value); } else { - N_N(_to_next_coro(gen, gen->path, event, value)); + PyObject *res = PyTuple_Pack(3, prefix, event, value); + CORO_SEND(gen->target_send, res); + Py_DECREF(res); } + Py_DECREF(prefix); Py_RETURN_NONE; } -PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *value) -{ - ParseBasecoro *gen = (ParseBasecoro *)self; - if (gen->join_path) { - return _send_impl_joint_path(gen, event, value); - } - return _send_impl(gen, event, value); -} - static PyObject* parse_basecoro_send(PyObject *self, PyObject *tuple) { PyObject *event = PyTuple_GetItem(tuple, 0); diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c/parse_basecoro.h python-ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.h --- python-ijson-3.1.3/ijson/backends/yajl2_c/parse_basecoro.h 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.h 2021-03-02 09:59:40.000000000 +0000 @@ -21,7 +21,6 @@ PyObject_HEAD PyObject *target_send; PyObject *path; - int join_path; } ParseBasecoro; /** diff -Nru python-ijson-3.1.3/ijson/backends/yajl2_c.py python-ijson-3.1.4/ijson/backends/yajl2_c.py --- python-ijson-3.1.3/ijson/backends/yajl2_c.py 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/backends/yajl2_c.py 2021-03-02 09:59:40.000000000 +0000 @@ -14,14 +14,6 @@ _get_buf_size = lambda kwargs: kwargs.pop('buf_size', 64 * 1024) -_get_prefix = lambda prefix: [] if not prefix else prefix.split(".") - -def _itemlike_pipeline(itemslike_coro, prefix, map_type, config): - return ( - (itemslike_coro, (_get_prefix(prefix), map_type), {}), - (_yajl2.parse_basecoro, (False,), {}), - (_yajl2.basic_parse_basecoro, [], config) - ) @utils.coroutine def basic_parse_basecoro(target, **kwargs): @@ -49,46 +41,30 @@ buf_size = _get_buf_size(kwargs) return _yajl2.parse_async(file, buf_size, **kwargs) - @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.kvitems_basecoro(target.send, prefix, map_type, **kwargs) -def kvitems_coro(target, prefix, map_type=None, **config): - return utils.chain(target.send, - *_itemlike_pipeline(_yajl2.kvitems_basecoro, prefix, map_type, config) - ) - def kvitems_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) - prefix = _get_prefix(prefix) return _yajl2.kvitems(f, buf_size, prefix, map_type, **kwargs) def kvitems_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) - prefix = _get_prefix(prefix) return _yajl2.kvitems_async(file, buf_size, prefix, map_type, **kwargs) - @utils.coroutine def items_basecoro(target, prefix, map_type=None, **kwargs): return _yajl2.items_basecoro(target.send, prefix, map_type, **kwargs) -def items_coro(target, prefix, map_type=None, **config): - return utils.chain(target.send, - *_itemlike_pipeline(_yajl2.items_basecoro, prefix, map_type, config) - ) - def items_gen(file, prefix, map_type=None, **kwargs): f = compat.bytes_reader(file) buf_size = _get_buf_size(kwargs) - prefix = _get_prefix(prefix) return _yajl2.items(f, buf_size, prefix, map_type, **kwargs) def items_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) - prefix = _get_prefix(prefix) return _yajl2.items_async(file, buf_size, prefix, map_type, **kwargs) common.enrich_backend(globals()) diff -Nru python-ijson-3.1.3/ijson/version.py python-ijson-3.1.4/ijson/version.py --- python-ijson-3.1.3/ijson/version.py 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/ijson/version.py 2021-03-02 09:59:40.000000000 +0000 @@ -1 +1 @@ -__version__ = '3.1.3' +__version__ = '3.1.4' diff -Nru python-ijson-3.1.3/README.rst python-ijson-3.1.4/README.rst --- python-ijson-3.1.3/README.rst 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/README.rst 2021-03-02 09:59:40.000000000 +0000 @@ -542,6 +542,8 @@ values greater than 2^32 in 32-bit platforms or Windows. Numbers with leading zeros are not reported as invalid (although they are invalid JSON numbers). + Incomplete JSON tokens at the end of an incomplete document + (e.g., ``{"a": fals``) are not reported as ``IncompleteJSONError``. * The ``python`` backend doesn't support ``allow_comments=True`` It also internally works with ``str`` objects, not ``bytes``, diff -Nru python-ijson-3.1.3/test/test_base.py python-ijson-3.1.4/test/test_base.py --- python-ijson-3.1.3/test/test_base.py 2020-11-24 02:46:01.000000000 +0000 +++ python-ijson-3.1.4/test/test_base.py 2021-03-02 09:59:40.000000000 +0000 @@ -237,6 +237,38 @@ b'{"key": "value"', b'{"key": "value",', ] +INCOMPLETE_JSON_TOKENS = [ + b'n', + b'nu', + b'nul', + b't', + b'tr', + b'tru', + b'f', + b'fa', + b'fal', + b'fals', + b'[f', + b'[fa', + b'[fal', + b'[fals', + b'[t', + b'[tr', + b'[tru', + b'[n', + b'[nu', + b'[nul', + b'{"key": t', + b'{"key": tr', + b'{"key": tru', + b'{"key": f', + b'{"key": fa', + b'{"key": fal', + b'{"key": fals', + b'{"key": n', + b'{"key": nu', + b'{"key": nul', +] STRINGS_JSON = br''' { "str1": "", @@ -342,6 +374,18 @@ self.assertEqual(2, len(ids)) self.assertListEqual([-2,-1], sorted(ids)) + def test_items_with_dotted_name(self): + json = b'{"0.1": 0}' + self.assertListEqual([0], self.get_all(self.items, json, '0.1')) + json = b'{"0.1": [{"a.b": 0}]}' + self.assertListEqual([0], self.get_all(self.items, json, '0.1.item.a.b')) + json = b'{"0.1": 0, "0": {"1": 1}}' + self.assertListEqual([0, 1], self.get_all(self.items, json, '0.1')) + json = b'{"abc.def": 0}' + self.assertListEqual([0], self.get_all(self.items, json, 'abc.def')) + self.assertListEqual([], self.get_all(self.items, json, 'abc')) + self.assertListEqual([], self.get_all(self.items, json, 'def')) + def test_map_type(self): obj = self.get_first(self.items, JSON, '') self.assertTrue(isinstance(obj, dict)) @@ -469,6 +513,13 @@ with self.assertRaises(common.IncompleteJSONError): self.get_all(self.basic_parse, json) + def test_incomplete_tokens(self): + if not self.handles_incomplete_json_tokens: + return + for json in INCOMPLETE_JSON_TOKENS: + with self.assertRaises(common.IncompleteJSONError): + self.get_all(self.basic_parse, json) + def test_invalid(self): for json in INVALID_JSONS: # Yajl1 doesn't complain about additional data after the end @@ -574,6 +625,7 @@ 'supports_multiple_values': name != 'yajl', 'supports_comments': name != 'python', 'detects_leading_zeros': name != 'yajl', + 'handles_incomplete_json_tokens': name != 'yajl' } return generate_backend_specific_tests(module, classname, method_suffix, members=members, *_bases) \ No newline at end of file