{% for region, css_class, title in regions %} {{ draw(region, source_size, css_class, title) }} @@ -254,7 +255,8 @@ a.nav.pull-left { margin-left: 0; } a.nav.pull-right { margin-right: 0; } h5 { margin-top: 40px; } - .annotated_image { position: relative; } + .annotated_image { position: relative; display: inline-block; } + .annotated_image img { max-width: 100%; } .region { position: absolute; } .source_region { outline: 2px solid #8080ff; } .region.matched { outline: 2px solid #ff0020; } diff -Nru stb-tester-30-5-gbefe47c/_stbt/match.py stb-tester-31/_stbt/match.py --- stb-tester-30-5-gbefe47c/_stbt/match.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/match.py 2019-09-18 14:04:32.000000000 +0000 @@ -7,10 +7,14 @@ License: LGPL v2.1 or (at your option) any later version (see https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). """ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import enum import itertools -import os from collections import namedtuple import cv2 @@ -22,7 +26,7 @@ from .imgutils import _frame_repr, _image_region, _load_image, crop, limit_time from .logging import ddebug, debug, draw_on, get_debug_level, ImageLogger from .sqdiff import sqdiff -from .types import Region, UITestFailure +from .types import Position, Region, UITestFailure class MatchMethod(enum.Enum): @@ -166,15 +170,6 @@ self.confirm_method, self.confirm_threshold, self.erode_passes)) -class Position(namedtuple('Position', 'x y')): - """A point within the video frame. - - `x` and `y` are integer coordinates (measured in number of pixels) from the - top left corner of the video frame. - """ - pass - - class MatchResult(object): """The result from `match`. @@ -222,7 +217,7 @@ "" if isinstance(self.image, numpy.ndarray) else repr(self.image))) - def __nonzero__(self): + def __bool__(self): return self.match @property @@ -382,6 +377,9 @@ if input_region is None: raise ValueError("frame with dimensions %r doesn't contain %r" % (frame.shape, region)) + if input_region.height < t.shape[0] or input_region.width < t.shape[1]: + raise ValueError("%r must be larger than reference image %r" + % (input_region, t.shape)) imglog = ImageLogger( "match", match_parameters=match_parameters, @@ -402,8 +400,7 @@ (template.relative_filename or template.image), first_pass_matched) imglog.append(matches=result) - draw_on(frame, result, label="match(%r)" % - os.path.basename(template.friendly_name)) + draw_on(frame, result, label="match(%s)" % template.short_repr()) yield result finally: @@ -490,7 +487,7 @@ self.expected, self.timeout_secs) -@memoize_iterator({"version": "30"}) +@memoize_iterator({"version": "31"}) def _find_matches(image, template, match_parameters, imglog): """Our image-matching algorithm. @@ -542,7 +539,7 @@ ddebug("Original image %s, template %s" % (image.shape, template.shape)) - method = { + method = { # pylint:disable=redefined-outer-name MatchMethod.SQDIFF: cv2.TM_SQDIFF, MatchMethod.SQDIFF_NORMED: cv2.TM_SQDIFF_NORMED, MatchMethod.CCORR_NORMED: cv2.TM_CCORR_NORMED, @@ -576,12 +573,12 @@ else: mask = None - template_pyramid = _build_pyramid(template, levels) + template_pyramid = _build_pyramid(template, levels, is_template=True) mask_pyramid = _build_pyramid(mask, len(template_pyramid), is_mask=True) - image_pyramid = _build_pyramid(image, len(template_pyramid)) + image_pyramid = _build_pyramid(image, len(mask_pyramid)) roi_mask = None # Initial region of interest: The whole image. - for level in reversed(range(len(template_pyramid))): + for level in reversed(range(len(image_pyramid))): if roi_mask is not None: if any(x < 3 for x in roi_mask.shape): roi_mask = None @@ -600,7 +597,7 @@ if level == 0: relax = 0 elif match_parameters.match_method == MatchMethod.SQDIFF: - relax = 0.01 + relax = 0.02 else: relax = 0.2 threshold = max(0, match_parameters.match_threshold - relax) @@ -651,7 +648,7 @@ width=template.shape[1], height=template.shape[0]) -def _match_template(image, template, mask, method, roi_mask, level, imwrite): +def _match_template(image, template, mask, method, roi_mask, level, imwrite): # pylint:disable=redefined-outer-name ddebug("Level %d: image %s, template %s" % ( level, image.shape, template.shape)) @@ -668,9 +665,20 @@ dtype=numpy.float32) if roi_mask is None: - rois = [ # Initial region of interest: The whole image. - Region(0, 0, matches_heatmap.shape[1], matches_heatmap.shape[0])] + # Initial region of interest: The whole image. + rois = [_image_region(matches_heatmap)] else: + ddebug("Level %d: roi_mask=%r, matches_heatmap=%r" % ( + level, roi_mask.shape, matches_heatmap.shape)) + + # roi_mask comes from the previous pyramid level so it has gone through + # pyrDown -> pyrUp. If the starting size was odd-numbered then after + # this round-trip you end up with a size 1 pixel larger than the + # original. We can discard the extra pixel safely because pyrUp blurs + # with a 5x5 kernel after upscaling, so in effect it is dilating all + # our ROIs by 1 pixel. + roi_mask = crop(roi_mask, _image_region(matches_heatmap)) + rois = [Region(*x) for x in cv2_compat.find_contour_boxes( roi_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)] _merge_regions(rois) @@ -697,7 +705,7 @@ for roi in rois: r = roi.extend(right=template.shape[1] - 1, bottom=template.shape[0] - 1) - ddebug("Level %d: Searching in %s" % (level, roi)) + ddebug("Level %d: Searching in %s" % (level, r)) cv2.matchTemplate( image[r.to_slice()], template, @@ -745,7 +753,7 @@ return (matched, best_match_position, certainty) -def _build_pyramid(image, levels, is_mask=False): +def _build_pyramid(image, levels, is_template=False, is_mask=False): """A "pyramid" is [an image, the same image at 1/2 the size, at 1/4, ...] As a performance optimisation, image processing algorithms work on a @@ -754,18 +762,36 @@ See http://docs.opencv.org/doc/tutorials/imgproc/pyramids/pyramids.html The original-sized image is called "level 0", the next smaller image "level - 1", and so on. This numbering corresponds to the array index of the - "pyramid" array. + 1", and so on. This numbering corresponds to the index of the "pyramid" + array. """ if image is None: return [None] * levels pyramid = [image] + previous = image for _ in range(levels - 1): - if any(x < 20 for x in pyramid[-1].shape[:2]): + if any(x < 20 for x in previous.shape[:2]): break - downsampled = cv2.pyrDown(pyramid[-1]) + downsampled = cv2.pyrDown(previous, borderType=cv2.BORDER_REPLICATE) if is_mask: cv2.threshold(downsampled, 254, 255, cv2.THRESH_BINARY, downsampled) + previous = downsampled + if is_template or is_mask: + # Ignore pixels on the edge of the template, because pyrDown's + # blurring will affect them differently than the corresponding + # pixels in the frame (which do have neighbours, unlike the + # template's edge pixels). + downsampled = downsampled[1:-1, 1:-1] + else: + # ...and adjust the coordinate system of the match positions we're + # returning accordingly. Because of the cropping, match position + # 0,0 means pixel 1,1 of the template matched at pixel 1,1 of + # the frame (this is the same as saying pixel 0,0 of the template + # matched at pixel 0,0 of the frame, so we won't need to adjust + # the match position afterwards). + downsampled = downsampled[1:, 1:] + if is_mask and numpy.count_nonzero(downsampled) == 0: + break pyramid.append(downsampled) return pyramid diff -Nru stb-tester-30-5-gbefe47c/_stbt/motion.py stb-tester-31/_stbt/motion.py --- stb-tester-30-5-gbefe47c/_stbt/motion.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/motion.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,11 @@ # coding: utf-8 +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + from collections import deque import cv2 @@ -16,8 +22,7 @@ """Generator that yields a sequence of one `MotionResult` for each frame processed from the device-under-test's video stream. - The `MotionResult` indicates whether any motion was detected -- that is, - any difference between two consecutive frames. + The `MotionResult` indicates whether any motion was detected. Use it in a ``for`` loop like this:: @@ -85,7 +90,10 @@ mask = _load_image(mask, cv2.IMREAD_GRAYSCALE) debug("Using mask %s" % mask.friendly_name) - frame = next(frames) + try: + frame = next(frames) + except StopIteration: + return region = Region.intersect(_image_region(frame), region) @@ -109,7 +117,6 @@ imglog.imwrite("previous_frame_gray", previous_frame_gray) absdiff = cv2.absdiff(frame_gray, previous_frame_gray) - previous_frame_gray = frame_gray imglog.imwrite("absdiff", absdiff) if mask.image is not None: @@ -134,6 +141,13 @@ out_region = out_region.translate(region.x, region.y) motion = bool(out_region) + if motion: + # Only update the comparison frame if it's different to the previous + # one. This makes `detect_motion` more sensitive to slow motion + # because the differences between frames 1 and 2 might be small and + # the differences between frames 2 and 3 might be small but we'd see + # the difference by looking between 1 and 3. + previous_frame_gray = frame_gray result = MotionResult(getattr(frame, "time", None), motion, out_region, frame) @@ -149,7 +163,7 @@ noise_threshold=None, mask=None, region=Region.ALL, frames=None): """Search for motion in the device-under-test's video stream. - "Motion" is difference in pixel values between two consecutive frames. + "Motion" is difference in pixel values between two frames. :type timeout_secs: int or float or None :param timeout_secs: @@ -263,7 +277,7 @@ self.region = region self.frame = frame - def __nonzero__(self): + def __bool__(self): return self.motion def __repr__(self): diff -Nru stb-tester-30-5-gbefe47c/_stbt/ocr.py stb-tester-31/_stbt/ocr.py --- stb-tester-30-5-gbefe47c/_stbt/ocr.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/ocr.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,11 @@ # coding: utf-8 +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + import errno import glob import os @@ -10,14 +16,13 @@ import cv2 import numpy -from kitchen.text.converters import to_bytes from . import imgproc_cache from .config import get_config from .imgutils import _frame_repr, _image_region, crop from .logging import debug, ImageLogger, warn from .types import Region -from .utils import named_temporary_directory +from .utils import named_temporary_directory, to_unicode # Tesseract sometimes has a hard job distinguishing certain glyphs such as # ligatures and different forms of the same punctuation. We strip out this @@ -77,13 +82,16 @@ class OcrEngine(IntEnum): - #: Tesseract's "legacy" OCR engine (v3). + #: Tesseract's "legacy" OCR engine (v3). Recommended. TESSERACT = 0 - #: Tesseract v4's "Long Short-Term Memory" neural network. + #: Tesseract v4's "Long Short-Term Memory" neural network. Not recommended + #: for reading menus, buttons, prices, numbers, times, etc, because it + #: hallucinates text that isn't there when the input isn't long prose. LSTM = 1 - #: Combine results from Tesseract legacy & LSTM engines. + #: Combine results from Tesseract legacy & LSTM engines. Not recommended + #: because it favours the result from the LSTM engine too heavily. TESSERACT_AND_LSTM = 2 #: Default engine, based on what is installed. @@ -123,7 +131,7 @@ self.text = text # pylint:disable=no-member - def __nonzero__(self): + def __bool__(self): return self.match def __repr__(self): @@ -141,7 +149,7 @@ mode=OcrMode.PAGE_SEGMENTATION_WITHOUT_OSD, lang=None, tesseract_config=None, tesseract_user_words=None, tesseract_user_patterns=None, upsample=True, text_color=None, - text_color_threshold=None, engine=None): + text_color_threshold=None, engine=None, char_whitelist=None): r"""Return the text present in the video frame as a Unicode string. Perform OCR (Optical Character Recognition) using the "Tesseract" @@ -224,9 +232,17 @@ section of :ref:`.stbt.conf`. :type engine: `OcrEngine` + :type char_whitelist: unicode string + :param char_whitelist: + String of characters that are allowed. Useful when you know that the + text is only going to contain numbers or IP addresses, for example so + that tesseract won't think that a zero is the letter o. + Note that Tesseract 4.0's LSTM engine ignores ``char_whitelist``. + | Added in v28: The ``upsample`` and ``text_color`` parameters. | Added in v29: The ``text_color_threshold`` parameter. | Added in v30: The ``engine`` parameter and support for Tesseract v4. + | Added in v31: The ``char_whitelist`` parameter. """ if frame is None: import stbt @@ -239,18 +255,18 @@ "instead. To OCR an entire video frame, use " "`region=Region.ALL`.") - if isinstance(tesseract_user_words, (str, unicode)): + if isinstance(tesseract_user_words, (bytes, str)): tesseract_user_words = [tesseract_user_words] - if isinstance(tesseract_user_patterns, (str, unicode)): + if isinstance(tesseract_user_patterns, (bytes, str)): tesseract_user_patterns = [tesseract_user_patterns] - imglog = ImageLogger("ocr") + imglog = ImageLogger("ocr", result=None) text, region = _tesseract( frame, region, mode, lang, tesseract_config, tesseract_user_patterns, tesseract_user_words, upsample, text_color, - text_color_threshold, engine, imglog) + text_color_threshold, engine, char_whitelist, imglog) text = text.strip().translate(_ocr_transtab) debug(u"OCR in region %s read '%s'." % (region, text)) _log_ocr_image_debug(imglog, text) @@ -261,7 +277,7 @@ mode=OcrMode.PAGE_SEGMENTATION_WITHOUT_OSD, lang=None, tesseract_config=None, case_sensitive=False, upsample=True, text_color=None, text_color_threshold=None, - engine=None): + engine=None, char_whitelist=None): """Search for the specified text in a single video frame. This can be used as an alternative to `match`, searching for text instead @@ -277,6 +293,7 @@ :param text_color: See `ocr`. :param text_color_threshold: See `ocr`. :param engine: See `ocr`. + :param char_whitelist: See `ocr`. :param bool case_sensitive: Ignore case if False (the default). :returns: @@ -294,6 +311,7 @@ | Added in v28: The ``upsample`` and ``text_color`` parameters. | Added in v29: The ``text_color_threshold`` parameter. | Added in v30: The ``engine`` parameter and support for Tesseract v4. + | Added in v31: The ``char_whitelist`` parameter. """ import lxml.etree if frame is None: @@ -309,13 +327,14 @@ xml, region = _tesseract(frame, region, mode, lang, _config, None, text.split(), upsample, text_color, - text_color_threshold, engine, imglog) + text_color_threshold, engine, char_whitelist, + imglog) if xml == '': hocr = None result = TextMatchResult(rts, False, None, frame, text) else: hocr = lxml.etree.fromstring(xml.encode('utf-8')) - p = _hocr_find_phrase(hocr, _to_unicode(text).split(), case_sensitive) + p = _hocr_find_phrase(hocr, to_unicode(text).split(), case_sensitive) if p: # Find bounding box box = None @@ -369,7 +388,8 @@ if _memoise_tesseract_version is None: try: _memoise_tesseract_version = subprocess.check_output( - ['tesseract', '--version'], stderr=subprocess.STDOUT) + ['tesseract', '--version'], + stderr=subprocess.STDOUT).decode("utf-8") except OSError as e: if e.errno == errno.ENOENT: return None @@ -382,7 +402,8 @@ def _tesseract(frame, region, mode, lang, _config, user_patterns, user_words, - upsample, text_color, text_color_threshold, engine, imglog): + upsample, text_color, text_color_threshold, engine, + char_whitelist, imglog): if _config is None: _config = {} @@ -414,9 +435,11 @@ imglog.imwrite("source", frame) imglog.set(engine=engine, mode=mode, lang=lang, + tesseract_config=_config.copy(), user_patterns=user_patterns, user_words=user_words, upsample=upsample, text_color=text_color, text_color_threshold=text_color_threshold, + char_whitelist=char_whitelist, tesseract_version=tesseract_version) frame_region = _image_region(frame) @@ -433,14 +456,15 @@ return (_tesseract_subprocess(crop(frame, region), mode, lang, _config, user_patterns, user_words, upsample, text_color, text_color_threshold, engine, - imglog, tesseract_version), + char_whitelist, imglog, tesseract_version), region) -@imgproc_cache.memoize({"version": "30"}) +@imgproc_cache.memoize({"version": "31"}) def _tesseract_subprocess( frame, mode, lang, _config, user_patterns, user_words, upsample, - text_color, text_color_threshold, engine, imglog, tesseract_version): + text_color, text_color_threshold, engine, char_whitelist, + imglog, tesseract_version): if tesseract_version >= LooseVersion("4.0"): engine_flags = ["--oem", str(int(engine))] @@ -464,7 +488,7 @@ diff = numpy.subtract(frame, text_color, dtype=numpy.int32) frame = numpy.sqrt((diff[:, :, 0] ** 2 + diff[:, :, 1] ** 2 + - diff[:, :, 2] ** 2) / 3) \ + diff[:, :, 2] ** 2) // 3) \ .astype(numpy.uint8) imglog.imwrite("text_color_difference", frame) _, frame = cv2.threshold(frame, text_color_threshold, 255, @@ -488,7 +512,8 @@ tessenv = os.environ.copy() - if _config or user_words or user_patterns or imglog.enabled: + if (_config or user_words or user_patterns or char_whitelist or + imglog.enabled): tessdata_dir = tmp + '/tessdata' os.mkdir(tessdata_dir) _symlink_copy_dir(_find_tessdata_dir(tessdata_suffix), tmp) @@ -504,30 +529,40 @@ if 'user_words_suffix' in _config: raise ValueError( "You cannot specify 'user_words' and " + - "'_config[\"user_words_suffix\"]' at the same time") + "'tesseract_config[\"user_words_suffix\"]' " + + "at the same time") with open('%s/%s.user-words' % (tessdata_dir, lang), 'w') as f: - f.write('\n'.join(to_bytes(x) for x in user_words)) + f.write('\n'.join(to_unicode(x) for x in user_words)) _config['user_words_suffix'] = 'user-words' if user_patterns: if 'user_patterns_suffix' in _config: raise ValueError( "You cannot specify 'user_patterns' and " + - "'_config[\"user_patterns_suffix\"]' at the same time") + "'tesseract_config[\"user_patterns_suffix\"]' " + + "at the same time") with open('%s/%s.user-patterns' % (tessdata_dir, lang), 'w') as f: - f.write('\n'.join(to_bytes(x) for x in user_patterns)) + f.write('\n'.join(to_unicode(x) for x in user_patterns)) _config['user_patterns_suffix'] = 'user-patterns' + if char_whitelist: + if 'tessedit_char_whitelist' in _config: + raise ValueError( + "You cannot specify 'char_whitelist' and " + + "'tesseract_config[\"tessedit_char_whitelist\"]' " + + "at the same time") + _config["tessedit_char_whitelist"] = char_whitelist + if imglog.enabled: _config['tessedit_write_images'] = True if _config: with open(tessdata_dir + '/configs/stbtester', 'w') as cfg: - for k, v in _config.iteritems(): + for k, v in _config.items(): if isinstance(v, bool): cfg.write(('%s %s\n' % (k, 'T' if v else 'F'))) else: - cfg.write("%s %s\n" % (k, to_bytes(v))) + cfg.write("%s %s\n" % (k, to_unicode(v))) cmd += ['stbtester'] cv2.imwrite(tmp + '/input.png', frame) @@ -535,7 +570,7 @@ subprocess.check_output(cmd, cwd=tmp, env=tessenv, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - warn("Tesseract failed: %s" % e.output) + warn("Tesseract failed: %s" % e.output.decode("utf-8", "replace")) raise if imglog.enabled: @@ -547,7 +582,7 @@ _, ext = os.path.splitext(filename) if ext == ".txt" or ext == ".hocr": with open(filename) as f: - return f.read().decode("utf-8") + return f.read() def _hocr_iterate(hocr): @@ -567,7 +602,7 @@ if need_space and started: yield (u' ', None) need_space = False - yield (unicode(t).strip(), e) + yield (str(t).strip(), e) started = True else: need_space = True @@ -592,13 +627,6 @@ return None -def _to_unicode(text): - if isinstance(text, str): - return text.decode("utf-8") - else: - return unicode(text) - - def _hocr_elem_region(elem): while elem is not None: m = re.search(r'bbox (\d+) (\d+) (\d+) (\d+)', elem.get('title') or u'') @@ -653,7 +681,12 @@ {{ annotated_image(result) }} -

Text:

+ {% if match_text %} +

Result:

{{ result | escape }}

+ {% endif %} + +

Tesseract output:

{{ output | escape }}

Parameters:

@@ -661,9 +694,11 @@ {% if match_text %}

case_sensitive={{case_sensitive}} {% endif %} +

char_whitelist={{char_whitelist}}

engine={{engine}}

lang={{lang}}

mode={{mode}} +

tesseract_config={{tesseract_config}}

tesseract_user_patterns={{user_patterns}}

tesseract_user_words={{user_words}}

tesseract_version={{tesseract_version}} diff -Nru stb-tester-30-5-gbefe47c/_stbt/power.py stb-tester-31/_stbt/power.py --- stb-tester-30-5-gbefe47c/_stbt/power.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/power.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import errno import os import re @@ -73,7 +78,7 @@ def get(self): import subprocess power = subprocess.check_output(self.cmd + ["status"]).strip() - return {'ON': True, 'OFF': False}[power] + return {b'ON': True, b'OFF': False}[power] class _Aviosys8800Pro(object): diff -Nru stb-tester-30-5-gbefe47c/_stbt/pylint_plugin.py stb-tester-31/_stbt/pylint_plugin.py --- stb-tester-30-5-gbefe47c/_stbt/pylint_plugin.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/pylint_plugin.py 2019-09-18 14:04:32.000000000 +0000 @@ -9,13 +9,20 @@ * http://docs.python.org/2/library/ast.html """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order +from future.utils import string_types import os import re import subprocess -from astroid import YES -from astroid.node_classes import BinOp, Call, Expr, Keyword +from astroid import MANAGER, YES +from astroid.node_classes import ( + Assert, BinOp, Call, Const, Expr, Keyword, Name, Raise) from astroid.scoped_nodes import ClassDef, FunctionDef from pylint.checkers import BaseChecker from pylint.interfaces import IAstroidChecker @@ -26,7 +33,7 @@ name = 'stb-tester' msgs = { # Range 70xx reserved for custom checkers: www.logilab.org/ticket/68057 - # When you add a new checker update the docstring in ../stbt-lint + # When you add a new checker update the docstring in ../stbt_lint.py 'E7001': ('Image "%s" not found on disk', 'stbt-missing-image', 'The image path given to "stbt.match" ' @@ -49,10 +56,24 @@ 'The image path given to "stbt.match" ' '(and similar functions) exists on disk, ' "but isn't committed to git."), + 'E7006': ('FrameObject properties must use "self._frame", not ' + '"get_frame()"', + 'stbt-frame-object-get-frame', + 'FrameObject properties must use "self._frame", not ' + '"stbt.get_frame()".'), + 'E7007': ('FrameObject properties must not use "%s"', + 'stbt-frame-object-property-press', + 'FrameObject properties must not have side-effects that ' + 'change the state of the device-under-test by calling ' + '"stbt.press()" or "stbt.press_and_wait()".'), + 'E7008': ('"assert True" has no effect', + 'stbt-assert-true', + '"assert True" has no effect; consider replacing it with a ' + 'comment or a call to "logging.info()".'), } def visit_const(self, node): - if (isinstance(node.value, str) and + if (isinstance(node.value, string_types) and re.search(r'.+\.png$', node.value) and "\n" not in node.value and not _is_uri(node.value) and @@ -87,6 +108,14 @@ self.add_message('E7003', node=node, args=arg.as_string()) if _in_frameobject(node) and _in_property(node): + if re.search(r"\bget_frame$", node.func.as_string()): + self.add_message('E7006', node=node) + + if re.search( + r"\b(press|press_and_wait|pressing|press_until_match)$", + node.func.as_string()): + self.add_message('E7007', node=node, args=node.func.as_string()) + for funcdef in _infer(node.func): argnames = _get_argnames(funcdef) if "frame" in argnames: @@ -102,6 +131,32 @@ self.add_message('E7004', node=node, args=node.as_string()) + def visit_assert(self, assertion): + if isinstance(assertion.test, Const) and assertion.test.value is True: + if assertion.fail: + self.add_message("E7008", node=assertion) + else: + self.add_message("E7008", node=assertion) + + +def _transform_assert_false_into_raise(assertion): + if isinstance(assertion.test, Const) and assertion.test.value is False: + out = Raise(lineno=assertion.lineno, + col_offset=assertion.col_offset, + parent=assertion.parent) + exc = Call(parent=out) + if assertion.fail: + args = [assertion.fail] + args[0].parent = exc + else: + args = [] + exc.postinit(Name("AssertionError", parent=exc), args) + out.postinit(exc, None) + return out + + +MANAGER.register_transform(Assert, _transform_assert_false_into_raise) + def _is_callable(node): failed_to_infer = True @@ -129,7 +184,8 @@ def _in_property(node): while node is not None: if isinstance(node, FunctionDef): - if "__builtin__.property" in node.decoratornames(): + if ("__builtin__.property" in node.decoratornames() or + "builtins.property" in node.decoratornames()): return True node = node.parent return False @@ -139,7 +195,7 @@ if isinstance(node, FunctionDef): return node.argnames() if isinstance(node, ClassDef) and node.newstyle: - for method in node.methods(): + for method in node.methods(): # pylint:disable=redefined-outer-name if method.name == "__init__": return method.argnames()[1:] return [] diff -Nru stb-tester-30-5-gbefe47c/_stbt/sqdiff.py stb-tester-31/_stbt/sqdiff.py --- stb-tester-30-5-gbefe47c/_stbt/sqdiff.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/sqdiff.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import ctypes import os @@ -185,8 +190,8 @@ _sqdiff_numba = _make_sqdiff_numba() - print "All times in ms numpy\tnumba" - print "type \tnumpy\tnumba\tC\tspeedup\tspeedup\tsize\talignment" + print("All times in ms numpy\tnumba") + print("type \tnumpy\tnumba\tC\tspeedup\tspeedup\tsize\talignment") for _ in range(100): frame_cropped, template, template_transparent = _random_template() @@ -207,8 +212,8 @@ repeat=3, number=10)) / 10 else: numba_time = float('nan') - print "%s\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%i x %i \t%s" % ( + print("%s\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%i x %i \t%s" % ( l, np_time * 1000, numba_time * 1000, c_time * 1000, np_time / c_time, numba_time / c_time, frame_cropped.shape[1], frame_cropped.shape[0], - frame_cropped.ctypes.data % 8) + frame_cropped.ctypes.data % 8)) diff -Nru stb-tester-30-5-gbefe47c/_stbt/stbt.conf stb-tester-31/_stbt/stbt.conf --- stb-tester-30-5-gbefe47c/_stbt/stbt.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/stbt.conf 2019-09-18 14:04:32.000000000 +0000 @@ -5,33 +5,11 @@ # Uncomment the following line to see the video while running stbt run: # sink_pipeline=xvimagesink sync=false -transformation_pipeline = identity control=error verbose=0 power_outlet=none v4l2_ctls= -# Handle loss of video (but without end-of-stream event) from the video capture -# device. Set to "True" if you're using the Hauppauge HD PVR. -restart_source = False - -# Send EOS before stopping source pipeline, to work around a bug in decklinksrc. -# Set to "True" if you're using a Blackmagic capture card and it hangs when the -# test ends. -source_teardown_eos = False - -# stbt camera settings that have to be in global as Python's configparse -# doesn't allow substitutions between sections. -geometriccorrection_params = -contraststretch_params = - -[camera] -# Format of the calibration videos that are generated by "stbt camera -# calibrate" to be played on your TV as part of the camera calibration process. -# Valid values are "ts" (for MPEG-TS) and "mp4". This is configurable because -# my Panasonic TV doesn't like MPEG-TS and my Samsung TV doesn't like MP4. -video_format = mp4 - [match] match_method=sqdiff match_threshold=0.98 @@ -70,9 +48,3 @@ [record] output_file=test.py control_recorder=file:///dev/stdin - -[batch] -pre_run = -post_run = -classify = -recover = diff -Nru stb-tester-30-5-gbefe47c/_stbt/stbt-power.sh stb-tester-31/_stbt/stbt-power.sh --- stb-tester-30-5-gbefe47c/_stbt/stbt-power.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/stbt-power.sh 2019-09-18 14:04:32.000000000 +0000 @@ -48,7 +48,7 @@ [[ "$command" =~ ^(on|off|status)$ ]] || die "invalid command '$command'" [[ -z "$uri" ]] && { - uri=$("$(dirname "$0")"/stbt-config "global.power_outlet" 2>/dev/null) || + uri=$("$(dirname "$0")"/stbt_config.py "global.power_outlet" 2>/dev/null) || die "no power-outlet specified on command line or in config file" } model=$(uri model "$uri") || die "invalid power-outlet uri '$uri'" diff -Nru stb-tester-30-5-gbefe47c/_stbt/stbt_run.py stb-tester-31/_stbt/stbt_run.py --- stb-tester-30-5-gbefe47c/_stbt/stbt_run.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/stbt_run.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import sys import traceback @@ -57,7 +62,7 @@ import_dir, import_name = find_import_name(filename_) sys.path.insert(0, import_dir) try: - module = import_module(import_name) + mod = import_module(import_name) finally: # If the test function is not in a module we will need to leave # PYTHONPATH modified here so one python file in the test-pack can @@ -65,7 +70,7 @@ # careful of modules that mess with sys.path: if '.' in import_name and sys.path[0] == import_dir: sys.path.pop(0) - return module + return mod _TestFunction = namedtuple( @@ -76,11 +81,11 @@ sys.argv = [script] + args if '::' in script: filename, funcname = script.split('::', 1) - module = _import_by_filename(filename) - function = getattr(module, funcname) + mod = _import_by_filename(filename) + func = getattr(mod, funcname) return _TestFunction( - script, filename, funcname, function.func_code.co_firstlineno, - function) + script, filename, funcname, func.__code__.co_firstlineno, + func) else: filename = os.path.abspath(script) @@ -103,44 +108,71 @@ def fn(): sys.path.insert(0, os.path.dirname(filename)) - execfile(filename, test_globals) + code = compile(open(filename, "rb").read(), + filename, + mode="exec", + # Don't apply the __future__ imports in force in + # this file. + dont_inherit=1) + exec(code, test_globals) # pylint:disable=exec-used return _TestFunction(script, script, "", 1, fn) -@contextmanager -def sane_unicode_and_exception_handling(script): - """ - Exit 1 on failure, and 2 on error. Print the traceback assuming UTF-8. - """ - # Simulates python3's defaulting to utf-8 output so we don't get confusing - # `UnicodeEncodeError`s when printing unicode characters: - from kitchen.text.converters import getwriter, exception_to_bytes, to_bytes - if sys.stdout.encoding is None: - sys.stdout = getwriter('utf8')(sys.stdout) - if sys.stderr.encoding is None: - sys.stderr = getwriter('utf8')(sys.stderr) +if sys.version_info.major == 2: # Python 2 - try: - yield - except Exception as e: # pylint:disable=broad-except - error_message = exception_to_bytes(e) - if not error_message and isinstance(e, AssertionError): - error_message = traceback.extract_tb(sys.exc_info()[2])[-1][3] - sys.stdout.write("FAIL: %s: %s: %s\n" % ( - script, type(e).__name__, error_message)) - - # This is a hack to allow printing exceptions that have unicode messages - # attached to them. The default behaviour of Python 2.7 is to replace - # unicode charactors with \x023-like backslash escapes. Instead we - # format them as utf-8 bytes - # - # It's not thread-safe, but will only be called at the end of execution: - traceback._some_str = to_bytes # pylint: disable=protected-access - traceback.print_exc(file=sys.stderr) - - # 1 is failure and 2 is error - if isinstance(e, (stbt.UITestFailure, AssertionError)): - sys.exit(1) # Failure - else: - sys.exit(2) # Error + @contextmanager + def sane_unicode_and_exception_handling(script): + """ + Exit 1 on failure, and 2 on error. Print the traceback assuming UTF-8. + """ + # Simulates python3's defaulting to utf-8 output so we don't get + # confusing `UnicodeEncodeError`s when printing unicode characters: + from kitchen.text.converters import ( # pylint:disable=import-error + getwriter, exception_to_bytes, to_bytes) + if sys.stdout.encoding is None: + sys.stdout = getwriter('utf8')(sys.stdout) + if sys.stderr.encoding is None: + sys.stderr = getwriter('utf8')(sys.stderr) + + try: + yield + except Exception as e: # pylint:disable=broad-except + error_message = exception_to_bytes(e) + if not error_message and isinstance(e, AssertionError): + error_message = traceback.extract_tb(sys.exc_info()[2])[-1][3] + sys.stdout.write(b"FAIL: %s: %s: %s\n" % ( + script, type(e).__name__, error_message)) + + # This is a hack to allow printing exceptions that have unicode + # messages attached to them. The default behaviour of Python 2.7 is + # to replace unicode charactors with \x023-like backslash escapes. + # Instead we format them as utf-8 bytes. + # + # It's not thread-safe, but will only be called at the end of + # execution: + traceback._some_str = to_bytes # pylint: disable=protected-access + traceback.print_exc(file=sys.stderr) + + if isinstance(e, (stbt.UITestFailure, AssertionError)): + sys.exit(1) # Failure + else: + sys.exit(2) # Error + +else: # Python 3 + + @contextmanager + def sane_unicode_and_exception_handling(script): + try: + yield + except Exception as e: # pylint:disable=broad-except + error_message = str(e) + if not error_message and isinstance(e, AssertionError): + error_message = traceback.extract_tb(sys.exc_info()[2])[-1][3] + sys.stdout.write("FAIL: %s: %s: %s\n" % ( + script, type(e).__name__, error_message)) + traceback.print_exc(file=sys.stderr) + if isinstance(e, (stbt.UITestFailure, AssertionError)): + sys.exit(1) # Failure + else: + sys.exit(2) # Error diff -Nru stb-tester-30-5-gbefe47c/_stbt/transition.py stb-tester-31/_stbt/transition.py --- stb-tester-30-5-gbefe47c/_stbt/transition.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/transition.py 2019-09-18 14:04:32.000000000 +0000 @@ -12,6 +12,11 @@ License: LGPL v2.1 or (at your option) any later version (see https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import enum @@ -19,7 +24,9 @@ import numpy from .core import load_image -from .logging import debug +from .imgutils import pixel_bounding_box +from .logging import ddebug, debug, draw_on +from .motion import MotionResult from .types import Region @@ -58,6 +65,7 @@ An object that will evaluate to true if the transition completed, false otherwise. It has the following attributes: + * **key** (*str*) – The name of the key that was pressed. * **frame** (`stbt.Frame`) – If successful, the first video frame when the transition completed; if timed out, the last frame seen. * **status** (`TransitionStatus`) – Either ``START_TIMEOUT``, @@ -172,12 +180,12 @@ "Transition didn't start within %s seconds of pressing %s", f, self.timeout_secs, press_result.key) return _TransitionResult( - f, TransitionStatus.START_TIMEOUT, + press_result.key, f, TransitionStatus.START_TIMEOUT, press_result.end_time, None, None) end_result = self.wait_for_transition_to_end(f) # pylint:disable=undefined-loop-variable return _TransitionResult( - end_result.frame, end_result.status, + press_result.key, end_result.frame, end_result.status, press_result.end_time, animation_start_time, end_result.end_time) def wait_for_transition_to_end(self, initial_frame): @@ -200,37 +208,68 @@ first_stable_frame, self.stable_secs, first_stable_frame.time) return _TransitionResult( - first_stable_frame, TransitionStatus.COMPLETE, + None, first_stable_frame, TransitionStatus.COMPLETE, None, initial_frame.time, first_stable_frame.time) if f.time >= self.expiry_time: _debug("Transition didn't end within %s seconds", f, self.timeout_secs) return _TransitionResult( - f, TransitionStatus.STABLE_TIMEOUT, + None, f, TransitionStatus.STABLE_TIMEOUT, None, initial_frame.time, None) def _debug(s, f, *args): - debug(("transition: %.3f: " + s) % ((f.time,) + args)) + debug(("transition: %.3f: " + s) % ((getattr(f, "time", 0),) + args)) -def strict_diff(f1, f2, region, mask_image): +def _ddebug(s, f, *args): + ddebug(("transition: %.3f: " + s) % ((getattr(f, "time", 0),) + args)) + + +def strict_diff(prev, frame, region, mask_image): if region is not None: - full_frame = Region(0, 0, f1.shape[1], f1.shape[0]) + full_frame = Region(0, 0, frame.shape[1], frame.shape[0]) region = Region.intersect(full_frame, region) - f1 = f1[region.y:region.bottom, region.x:region.right] - f2 = f2[region.y:region.bottom, region.x:region.right] + f1 = prev[region.y:region.bottom, region.x:region.right] + f2 = frame[region.y:region.bottom, region.x:region.right] absdiff = cv2.absdiff(f1, f2) if mask_image is not None: absdiff = cv2.bitwise_and(absdiff, mask_image, absdiff) - return numpy.count_nonzero(absdiff) > 50 or (absdiff > 20).any() + diffs_found = False + out_region = None + maxdiff = numpy.max(absdiff) + if maxdiff > 20: + diffs_found = True + big_diffs = absdiff > 20 + out_region = pixel_bounding_box(big_diffs) + _ddebug("found %s diffs above 20 (max %s) in %r", frame, + numpy.count_nonzero(big_diffs), maxdiff, out_region) + elif maxdiff > 0: + small_diffs = absdiff > 5 + small_diffs_count = numpy.count_nonzero(small_diffs) + if small_diffs_count > 50: + diffs_found = True + out_region = pixel_bounding_box(small_diffs) + _ddebug("found %s diffs <= %s in %r", frame, small_diffs_count, + maxdiff, out_region) + else: + _ddebug("only found %s diffs <= %s", frame, small_diffs_count, + maxdiff) + if out_region: + out_region = out_region.translate(region.x, region.y) + + result = MotionResult(getattr(frame, "time", None), diffs_found, + out_region, frame) + draw_on(frame, result, label="transition") + return result class _TransitionResult(object): - def __init__( - self, frame, status, press_time, animation_start_time, end_time): + def __init__(self, key, frame, status, press_time, animation_start_time, + end_time): + self.key = key self.frame = frame self.status = status self.press_time = press_time @@ -239,8 +278,9 @@ def __repr__(self): return ( - "_TransitionResult(frame=, status=%s, press_time=%s, " - "animation_start_time=%s, end_time=%s)" % ( + "_TransitionResult(key=%r, frame=, status=%s, " + "press_time=%s, animation_start_time=%s, end_time=%s)" % ( + self.key, self.status, self.press_time, self.animation_start_time, @@ -249,9 +289,10 @@ def __str__(self): # Also lists the properties -- it's useful to see them in the logs. return ( - "_TransitionResult(frame=, status=%s, press_time=%s, " - "animation_start_time=%s, end_time=%s, duration=%s, " + "_TransitionResult(key=%r, frame=, status=%s, " + "press_time=%s, animation_start_time=%s, end_time=%s, duration=%s, " "animation_duration=%s)" % ( + self.key, self.status, self.press_time, self.animation_start_time, @@ -259,7 +300,7 @@ self.duration, self.animation_duration)) - def __nonzero__(self): + def __bool__(self): return self.status == TransitionStatus.COMPLETE @property diff -Nru stb-tester-30-5-gbefe47c/_stbt/tv_driver.py stb-tester-31/_stbt/tv_driver.py --- stb-tester-30-5-gbefe47c/_stbt/tv_driver.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/tv_driver.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,219 +0,0 @@ -import os -import sys -from time import sleep - -from _stbt.utils import mkdir_p - - -def _gen_video_cache_dir(): - cache_root = (os.environ.get("XDG_CACHE_HOME", None) or - os.environ.get("HOME") + '/.cache') - return cache_root + '/stbt/camera-video-cache' - - -def _generate_video_if_not_exists(video, video_generator, format_): - from os.path import isfile - filename = "%s/%s.%s" % (_gen_video_cache_dir(), video, format_) - if not isfile(filename): - from _stbt.gst_utils import frames_to_video - import tempfile - sys.stderr.write( - "Creating test video '%s'. This only has to happen once but may " - "take some time...\n" % filename) - - # Create the video atomically to avoid serving invalid mp4s - tf = tempfile.NamedTemporaryFile(prefix=filename, delete=False) - frame_caps, frame_generator = video_generator[video] - frames_to_video( - tf.name, frame_generator(), caps=frame_caps, container=format_) - os.rename(tf.name, filename) - - sys.stderr.write("Test video generation complete.\n") - return filename - - -def _get_external_ip(): - import socket - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("192.0.2.0", 80)) - return s.getsockname()[0] - - -class _HTTPVideoServer(object): - def __init__(self, video_generators, video_format): - self._video_generators = dict(video_generators) - self._video_format = video_format - self._lighttpd_pid = None - self._base_url = None - - self._start() - - def _start(self): - from textwrap import dedent - from tempfile import NamedTemporaryFile - from subprocess import CalledProcessError, check_output, STDOUT - from random import randint - video_cache_dir = _gen_video_cache_dir() - mkdir_p(video_cache_dir) - lighttpd_config_file = NamedTemporaryFile( - prefix='stbt-camera-lighttpd-', suffix='.conf', delete=False) - pidfile = NamedTemporaryFile( - prefix="stbt-camera-lighttpd-", suffix=".pidfile") - # This is an awful way to start listening on a random port and not a - # great way of tracking the sub-process. - port = None - while port is None: - try: - lighttpd_config_file.seek(0) - lighttpd_config_file.truncate(0) - try_port = randint(10000, 30000) - lighttpd_config_file.write(dedent("""\ - # This file is generated automatically by stb-tester. - # DO NOT EDIT. - server.document-root = "%s" - - server.port = %i - - server.pid-file = "%s" - - mimetype.assign = ( - ".png" => "image/png", - ".mp4" => "video/mp4", - ".ts" => "video/MP2T" - )""") % (video_cache_dir, try_port, pidfile.name)) - lighttpd_config_file.flush() - check_output(['lighttpd', '-f', lighttpd_config_file.name], - close_fds=True, stderr=STDOUT) - port = try_port - except CalledProcessError as e: - if e.output.find('Address already in use') != -1: - pass - else: - sys.stderr.write("lighttpd failed to start: %s\n" % - e.output) - raise - # lighttpd writes its pidfile out after forking rather than before - # causing a race. The real fix is to patch lighttpd to support socket - # passing and then open the listening socket ourselves. - while os.fstat(pidfile.fileno()).st_size == 0: - sleep(0.1) - self._lighttpd_pid = int(pidfile.read()) - self._base_url = "http://%s:%i/" % (_get_external_ip(), port) - - @property - def mime_type(self): - return { - 'mp4': 'video/mp4', - 'ts': 'video/MP2T', - }[self._video_format] - - def __del__(self): - from signal import SIGTERM - from os import kill - if self._lighttpd_pid: - kill(self._lighttpd_pid, SIGTERM) - - def get_url(self, video): - _generate_video_if_not_exists(video, self._video_generators, - self._video_format) - return "%s%s.%s" % (self._base_url, video, self._video_format) - - -class _AssumeTvDriver(object): - def show(self, filename): - sys.stderr.write("Assuming video %s is playing\n" % filename) - - def stop(self): - sys.stderr.write("Assuming videos are no longer playing\n") - - -class _FakeTvDriver(object): - """TV driver intended to be paired up with fake-video-src.py from the test - directory""" - def __init__(self, control_pipe, video_server): - self.control_pipe = open(control_pipe, 'w') - self.video_server = video_server - - def show(self, video): - uri = self.video_server.get_url(video) - self.control_pipe.write("%s\n" % uri) - self.control_pipe.flush() - - def stop(self): - self.control_pipe.write("stop\n") - self.control_pipe.flush() - - -class _ManualTvDriver(object): - def __init__(self, video_server): - self.video_server = video_server - - def show(self, video): - url = self.video_server.get_url(video) - sys.stderr.write( - "Please show %s video. This can be found at %s\n" % (video, url) + - "\n" + - "Press when video is showing\n") - sys.stdin.readline() - sys.stderr.write("Thank you\n") - - def stop(self): - sys.stderr.write("Please return TV back to original state\n") - - -class _AdbTvDriver(object): - def __init__(self, video_server, adb_cmd=None): - if adb_cmd is None: - adb_cmd = ['adb'] - self.adb_cmd = adb_cmd - self.video_server = video_server - - def show(self, video): - import subprocess - cmd = self.adb_cmd + [ - 'shell', 'am', 'start', - '-a', 'android.intent.action.VIEW', - '-d', self.video_server.get_url(video), - '-t', self.video_server.mime_type] - subprocess.check_call(cmd, close_fds=True) - - def stop(self): - pass - - -def add_argparse_argument(argparser): - from .config import get_config - argparser.add_argument( - "--tv-driver", - help="Determines how to display videos on TV.\n\n" - " manual - Prompt the user then wait for confirmation.\n" - " assume - Assume the video is already playing (useful for " - "scripting when passing a single test to be run).\n" - " fake:pipe_name - Used for testing\n" - " adb[:adb_command] - Control an android device over adb", - default=get_config("camera", "tv_driver", "manual")) - - -def create_from_args(args, video_generator): - from .config import get_config - return create_from_description( - args.tv_driver, video_generator, get_config('camera', 'video_format')) - - -def create_from_description(desc, video_generator, video_format): - def make_video_server(): - return _HTTPVideoServer(video_generator, video_format=video_format) - - if desc == 'assume': - return _AssumeTvDriver() - elif desc.startswith('fake:'): - return _FakeTvDriver(desc[5:], make_video_server()) - elif desc == 'manual': - return _ManualTvDriver(make_video_server()) - elif desc == 'adb': - return _AdbTvDriver(make_video_server()) - elif desc.startswith('adb:'): - import shlex - return _AdbTvDriver(make_video_server(), shlex.split(desc[4:])) - else: - raise RuntimeError("Unknown video driver requested: %s" % desc) diff -Nru stb-tester-30-5-gbefe47c/_stbt/types.py stb-tester-31/_stbt/types.py --- stb-tester-30-5-gbefe47c/_stbt/types.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/types.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,11 +1,63 @@ # coding: utf-8 # Don't import anything not in the Python standard library from this file +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order +from future.utils import with_metaclass + from collections import namedtuple -class Region(namedtuple('Region', 'x y right bottom')): - u""" +class Position(namedtuple('Position', 'x y')): + """A point with ``x`` and ``y`` coordinates.""" + pass + + +class _RegionClsMethods(type): + """Metaclass for `Region`. + + This defines some classmethods for Region, but in a way that they can't be + called on an instance of Region (which is an easy mistake to make, but with + incorrect behaviour). See . + """ + + def intersect(cls, *args): + out = Region.ALL + args = iter(args) + try: + out = next(args) + except StopIteration: + # No arguments passed: + return Region.ALL + if out is None: + return None + + for r in args: + if not r: + return None + out = (max(out[0], r[0]), max(out[1], r[1]), + min(out[2], r[2]), min(out[3], r[3])) + if out[0] >= out[2] or out[1] >= out[3]: + return None + return Region.from_extents(*out) + + def bounding_box(cls, *args): + args = [_f for _f in args if _f] + if not args: + return None + return Region.from_extents( + min(r.x for r in args), + min(r.y for r in args), + max(r.right for r in args), + max(r.bottom for r in args)) + + +class Region(with_metaclass(_RegionClsMethods, + namedtuple('Region', 'x y right bottom'))): + r""" ``Region(x, y, width=width, height=height)`` or ``Region(x, y, right=right, bottom=bottom)`` @@ -48,9 +100,9 @@ True >>> Region.intersect(c, b) == c True - >>> print Region.intersect(a, c) + >>> print(Region.intersect(a, c)) None - >>> print Region.intersect(None, a) + >>> print(Region.intersect(None, a)) None >>> Region.intersect(a) Region(x=0, y=0, right=8, bottom=8) @@ -65,7 +117,7 @@ True >>> Region.ALL Region.ALL - >>> print Region.ALL + >>> print(Region.ALL) Region.ALL >>> c.above() Region(x=10, y=-inf, right=13, bottom=4) @@ -98,8 +150,71 @@ The y coordinate of the bottom edge of the region, measured in pixels from the top of the video frame (exclusive). - ``x``, ``y``, ``right``, and ``bottom`` can be infinite -- that is, - ``float("inf")`` or ``-float("inf")``. + .. py:attribute:: width + + The width of the region, measured in pixels. + + .. py:attribute:: height + + The height of the region, measured in pixels. + + ``x``, ``y``, ``right``, ``bottom``, ``width`` and ``height`` can be + infinite --- that is, ``float("inf")`` or ``-float("inf")``. + + .. py:staticmethod:: from_extents + + Create a Region using right and bottom extents rather than width and + height. + + Typically you'd use the ``right`` and ``bottom`` parameters of the + ``Region`` constructor instead, but this factory function is useful + if you need to create a ``Region`` from a tuple. + + >>> extents = (4, 4, 13, 10) + >>> Region.from_extents(*extents) + Region(x=4, y=4, right=13, bottom=10) + + .. py:staticmethod:: bounding_box(*args) + + :returns: The smallest region that contains all the given regions. + + >>> a = Region(50, 20, right=60, bottom=40) + >>> b = Region(20, 30, right=30, bottom=50) + >>> c = Region(55, 25, right=70, bottom=35) + >>> Region.bounding_box(a, b) + Region(x=20, y=20, right=60, bottom=50) + >>> Region.bounding_box(b, b) + Region(x=20, y=30, right=30, bottom=50) + >>> Region.bounding_box(None, b) + Region(x=20, y=30, right=30, bottom=50) + >>> Region.bounding_box(b, None) + Region(x=20, y=30, right=30, bottom=50) + >>> Region.bounding_box(b, Region.ALL) + Region.ALL + >>> print(Region.bounding_box(None, None)) + None + >>> print(Region.bounding_box()) + None + >>> Region.bounding_box(b) + Region(x=20, y=30, right=30, bottom=50) + >>> Region.bounding_box(a, b, c) == \ + ... Region.bounding_box(a, Region.bounding_box(b, c)) + True + + Changed in v30: ``bounding_box`` can take an arbitrary number of region + arguments, rather than exactly two. + + .. py:staticmethod:: intersect(*args) + + :returns: The intersection of the passed regions, or ``None`` if the + regions don't intersect. + + Any parameter can be ``None`` (an empty Region) so intersect is + commutative and associative. + + Changed in v30: ``intersect`` can take an arbitrary number of region + arguments, rather than exactly two. + """ def __new__(cls, x, y, width=None, height=None, right=None, bottom=None): if (width is None) == (right is None): @@ -125,68 +240,36 @@ @property def width(self): - """The width of the region, measured in pixels.""" return self.right - self.x @property def height(self): - """The height of the region, measured in pixels.""" return self.bottom - self.y - def to_slice(self): - """A 2-dimensional slice suitable for indexing a `stbt.Frame`.""" - return (slice(self.y, self.bottom), slice(self.x, self.right)) - @staticmethod def from_extents(x, y, right, bottom): - """Create a Region using right and bottom extents rather than width and - height. - - Typically you'd use the ``right`` and ``bottom`` parameters of the - ``Region`` constructor instead, but this factory function is useful - if you need to create a ``Region`` from a tuple. - - >>> extents = (4, 4, 13, 10) - >>> Region.from_extents(*extents) - Region(x=4, y=4, right=13, bottom=10) - """ return Region(x, y, right=right, bottom=bottom) - @staticmethod - def intersect(*args): - """ - :returns: The intersection of the passed regions, or ``None`` if the - regions don't intersect. - - Any parameter can ``None`` so intersect is commutative and associative. - - New in v30: intersect can now take an arbitrary number of region - arguments rather than exactly two. - """ - out = Region.ALL - args = iter(args) - try: - out = next(args) - except StopIteration: - # No arguments passed: - return Region.ALL - if out is None: - return None - - for r in args: - if not r: - return None - out = (max(out[0], r[0]), max(out[1], r[1]), - min(out[2], r[2]), min(out[3], r[3])) - if out[0] >= out[2] or out[1] >= out[3]: - return None - return Region.from_extents(*out) + def to_slice(self): + """A 2-dimensional slice suitable for indexing a `stbt.Frame`.""" + return (slice(max(0, self.y), + max(0, self.bottom)), + slice(max(0, self.x), + max(0, self.right))) def contains(self, other): """:returns: True if ``other`` is entirely contained within self.""" return (other and self.x <= other.x and self.y <= other.y and self.right >= other.right and self.bottom >= other.bottom) + def translate(self, x=0, y=0): + """ + :returns: A new region with the position of the region adjusted by the + given amounts. + """ + return Region.from_extents(self.x + x, self.y + y, + self.right + x, self.bottom + y) + def extend(self, x=0, y=0, right=0, bottom=0): """ :returns: A new region with the edges of the region adjusted by the @@ -229,13 +312,26 @@ return Region(x=x, y=y, right=right, bottom=bottom) - def translate(self, x=0, y=0): + def dilate(self, n): + """Expand the region by n px in all directions. + + >>> Region(20, 30, right=30, bottom=50).dilate(3) + Region(x=17, y=27, right=33, bottom=53) """ - :returns: A new region with the position of the region adjusted by the - given amounts. + return self.extend(x=-n, y=-n, right=n, bottom=n) + + def erode(self, n): + """Shrink the region by n px in all directions. + + >>> Region(20, 30, right=30, bottom=50).erode(3) + Region(x=23, y=33, right=27, bottom=47) + >>> print(Region(20, 30, 10, 20).erode(5)) + None """ - return Region.from_extents(self.x + x, self.y + y, - self.right + x, self.bottom + y) + if self.width > n * 2 and self.height > n * 2: + return self.dilate(-n) + else: + return None def above(self, height=float('inf')): """ @@ -265,66 +361,6 @@ """ return self.replace(x=self.x - width, right=self.x) - @staticmethod - def bounding_box(*args): - r"""Find the bounding box of the given regions. Returns the smallest - region which contains all passed regions. - - >>> a = Region(50, 20, right=60, bottom=40) - >>> b = Region(20, 30, right=30, bottom=50) - >>> c = Region(55, 25, right=70, bottom=35) - >>> Region.bounding_box(a, b) - Region(x=20, y=20, right=60, bottom=50) - >>> Region.bounding_box(b, b) - Region(x=20, y=30, right=30, bottom=50) - >>> Region.bounding_box(None, b) - Region(x=20, y=30, right=30, bottom=50) - >>> Region.bounding_box(b, None) - Region(x=20, y=30, right=30, bottom=50) - >>> Region.bounding_box(b, Region.ALL) - Region.ALL - >>> print Region.bounding_box(None, None) - None - >>> print Region.bounding_box() - None - >>> Region.bounding_box(b) - Region(x=20, y=30, right=30, bottom=50) - >>> Region.bounding_box(a, b, c) == \ - ... Region.bounding_box(a, Region.bounding_box(b, c)) - True - - New in v30: No longer limited to just taking 2 regions. - """ - args = filter(None, args) - if not args: - return None - return Region.from_extents( - min(r.x for r in args), - min(r.y for r in args), - max(r.right for r in args), - max(r.bottom for r in args)) - - def dilate(self, n): - """Expand the region by n px in all directions. - - >>> Region(20, 30, right=30, bottom=50).dilate(3) - Region(x=17, y=27, right=33, bottom=53) - """ - return self.extend(x=-n, y=-n, right=n, bottom=n) - - def erode(self, n): - """Shrink the region by n px in all directions. - - >>> Region(20, 30, right=30, bottom=50).erode(3) - Region(x=23, y=33, right=27, bottom=47) - >>> print Region(20, 30, 10, 20).erode(5) - None - """ - if self.width > n * 2 and self.height > n * 2: - return self.dilate(-n) - else: - return None - Region.ALL = Region(x=-float('inf'), y=-float('inf'), right=float('inf'), bottom=float('inf')) diff -Nru stb-tester-30-5-gbefe47c/_stbt/utils.py stb-tester-31/_stbt/utils.py --- stb-tester-30-5-gbefe47c/_stbt/utils.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/utils.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import errno import os import tempfile @@ -11,7 +16,7 @@ exceptions""" try: os.makedirs(d) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(d) \ and os.access(d, os.R_OK | os.W_OK): return @@ -30,7 +35,7 @@ @contextmanager def named_temporary_directory( - suffix='', prefix='tmp', dir=None): # pylint:disable=redefined-builtin + suffix='', prefix='tmp', dir=None): # pylint:disable=redefined-builtin,redefined-outer-name dirname = tempfile.mkdtemp(suffix, prefix, dir) try: yield dirname @@ -49,6 +54,16 @@ os.chdir(olddir) +@contextmanager +def scoped_process(process): + try: + yield process + finally: + if process.poll() is None: + process.kill() + process.wait() + + def find_import_name(filename): """ To import an arbitrary filename we need to set PYTHONPATH and we need to @@ -71,3 +86,19 @@ import_dir, s = os.path.split(import_dir) import_name = "%s.%s" % (s, import_name) return import_dir, import_name + + +def to_bytes(text): + if isinstance(text, str): + return text.encode("utf-8", errors="backslashreplace") + elif isinstance(text, bytes): + return text + else: + raise TypeError("Unexpected type %s" % type(text)) + + +def to_unicode(text): + if isinstance(text, bytes): + return text.decode("utf-8", errors="replace") + else: + return str(text) diff -Nru stb-tester-30-5-gbefe47c/_stbt/x11.py stb-tester-31/_stbt/x11.py --- stb-tester-30-5-gbefe47c/_stbt/x11.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/x11.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,10 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os -import Queue +import queue import random import signal import subprocess @@ -22,7 +27,7 @@ > SIGUSR1 to its parent process after it has set up the various connection > schemes. """ - q = Queue.Queue() + q = queue.Queue() def on_signal(signo, _stack_frame): q.put(signo) diff -Nru stb-tester-30-5-gbefe47c/_stbt/xxhash.py stb-tester-31/_stbt/xxhash.py --- stb-tester-30-5-gbefe47c/_stbt/xxhash.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/_stbt/xxhash.py 2019-09-18 14:04:32.000000000 +0000 @@ -4,10 +4,16 @@ with xxhash takes ~242us, whereas using hash() builtin or hashlib.sha1 takes >3ms. """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import binascii import ctypes import os import struct +import sys _libxxhash = ctypes.CDLL( os.path.dirname(os.path.abspath(__file__)) + "/libxxhash.so") @@ -60,12 +66,15 @@ def update(self, data): # Passing a buffer/memoryview object via ctypes is inconvenient. See # http://thread.gmane.org/gmane.comp.python.devel/134936/focus=134941 - buf = buffer(data) + if sys.version_info.major == 2: # Python 2 + buf = buffer(data) # pylint:disable=undefined-variable + else: # Python 3 + buf = memoryview(data) address = ctypes.c_void_p() length = ctypes.c_ssize_t() ctypes.pythonapi.PyObject_AsReadBuffer( ctypes.py_object(buf), ctypes.byref(address), ctypes.byref(length)) - assert length >= 0 + assert length.value >= 0 _libxxhash.XXH64_update( self._state, address, ctypes.c_size_t(length.value)) @@ -74,4 +83,4 @@ return struct.pack(">Q", _libxxhash.XXH64_digest(self._state)) def hexdigest(self): - return binascii.hexlify(self.digest()) + return binascii.hexlify(self.digest()).decode("utf-8") diff -Nru stb-tester-30-5-gbefe47c/stbt_auto_selftest.py stb-tester-31/stbt_auto_selftest.py --- stb-tester-30-5-gbefe47c/stbt_auto_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt_auto_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,564 +0,0 @@ -#!/usr/bin/env python - -""" -``stbt auto-selftest`` captures the behaviour of Frame Objects (and other -helper functions that operate on a video-frame) by generating doctests. When -you change these helper functions, the generated doctests help to ensure that -they still behave correctly. - -Usage: - - stbt auto-selftest generate [source_file.py ...] - stbt auto-selftest validate - -``stbt auto-selftest generate`` generates a doctest for every `FrameObject` in -your test-pack against any screenshots stored in ``selftest/screenshots``. This -results in a set of python files under ``selftest/auto_selftest`` which can be -inspected by (human) eye and validated either with ``python -m doctest -`` or more commonly with ``stbt auto-selftest validate``. - -**auto-selftest checklist** - -1. Take screenshots of the device-under-test and put them in the - ``selftest/screenshots`` directory of your test-pack. -2. Run ``stbt auto-selftest generate`` after every change to your Frame - Objects, or after adding a new screenshot. -3. View the effect of your changes with ``git diff``. -4. Commit the changes to your auto-selftests along with your changes to the - Frame Objects. -5. Run ``stbt auto-selftest validate`` on every change from your Continuous - Integration system. - -Using auto-selftest makes it much easier to create, update and modify Frame -Objects. If you find a screen where your Frame Object doesn't behave properly, -add that screenshot to your selftest corpus, and fix the Frame Object; -auto-selftest will check that you haven't introduced a regression in the Frame -Object's behaviour against the other screenshots. - -For more information and for more advanced usage see the example test file -(``tests/example.py``), the accompanying screenshots -(``selftest/screenshots``), and the generated doctest -(``selftest/auto_selftest/tests/example_selftest.py``) under -. - -For more information on the background behind auto-selftests see -`Improve black-box testing agility: automatic self-regression tests -`_. -""" - -import argparse -import cStringIO -import errno -import fnmatch -import multiprocessing -import os -import re -import shutil -import signal -import StringIO -import sys -import tempfile -import time -import traceback -from collections import namedtuple -from textwrap import dedent, wrap - -from _stbt.imgproc_cache import cache -from _stbt.utils import find_import_name, mkdir_p, rm_f - -SCREENSHOTS_ROOT = "selftest/screenshots" - - -def main(argv): - parser = argparse.ArgumentParser( - prog="stbt auto-selftest", - epilog=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - subparsers = parser.add_subparsers(dest="command") - - subparser_generate = subparsers.add_parser( - 'generate', help="Regenerate auto-selftests from screenshots") - subparser_generate.add_argument( - "source_files", nargs="*", help="""Python source file(s) to search for - FrameObjects (defaults to all python files in the test-pack)""") - - subparsers.add_parser('validate', help='Run (and check) the auto-selftests') - - cmdline_args = parser.parse_args(argv[1:]) - - root = _find_test_pack_root() - if root is None: - sys.stderr.write( - "error: This command must be run within a test pack. Couldn't find " - "a .stbt.conf in this or any parent directory.\n") - return 1 - - os.chdir(root) - - if cmdline_args.command == 'generate': - return generate(cmdline_args.source_files) - elif cmdline_args.command == 'validate': - return validate() - else: - assert False - - -def generate(source_files): - tmpdir, modules = generate_into_tmpdir(source_files) - target_dir = os.path.join(os.curdir, "selftest/auto_selftest") - try: - if source_files: - # Only replace the selftests for the specified files. - for f in modules: - newfile = os.path.join(tmpdir, selftest_filename(f.filename)) - target = os.path.join(target_dir, selftest_filename(f.filename)) - if os.path.exists(newfile): - mkdir_p(os.path.dirname(target)) - os.rename(newfile, target) - else: - rm_f(target) - if f.error: - sys.stderr.write( - "error: '%s' isn't a valid source file.\n" - % f.filename) - return 1 - sys.stderr.write( - "warning: '%s' doesn't define any selftests.\n" - % f.filename) - prune_empty_directories(target_dir) - else: - # Replace all selftests, deleting selftests for source files that - # no longer exist. - if os.path.exists(target_dir): - shutil.rmtree(target_dir) - os.rename(tmpdir, target_dir) - - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - -def validate(): - import filecmp - tmpdir, _ = generate_into_tmpdir() - try: - orig_files = _recursive_glob( - '*.py', "%s/selftest/auto_selftest" % os.curdir) - new_files = _recursive_glob('*.py', tmpdir) - if orig_files != new_files: - return 1 - _, mismatch, errors = filecmp.cmpfiles( - tmpdir, "%s/selftest/auto_selftest" % os.curdir, orig_files) - if mismatch or errors: - return 1 - else: - return 0 - finally: - shutil.rmtree(tmpdir) - - -def is_valid_python_identifier(x): - return bool(re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', x)) - - -def prune_empty_directories(dir_): - for root, _dirs, files in os.walk(dir_, topdown=False): - if len(files) == 0 and root != dir_: - try: - os.rmdir(root) - except OSError as e: - if e.errno not in [errno.EEXIST, errno.ENOTEMPTY]: - raise - - -def init_worker(): - signal.signal(signal.SIGINT, signal.SIG_IGN) - - -def iterate_with_progress(sequence, width=20, stream=sys.stderr): - stream.write('\n') - total = len(sequence) - for n, v in enumerate(sequence): - stream.write(_progress_line(width, n, total) + - " - Processing %s\r" % v) - yield v - stream.write(_progress_line(width, total, total) + "\n") - - -def _progress_line(width, n, total): - ANSI_ERASE_LINE = '\033[K' - progress = (n * width) // total - return '\r' + ANSI_ERASE_LINE + '[%s] %3d / %d' % ( - '#' * progress + ' ' * (width - progress), n, total) - - -def generate_into_tmpdir(source_files=None): - start_time = time.time() - - # Importing stbt + gstreamer bindings + opencv is slow. - # multiprocessing.Pool uses `fork` so by importing here, these modules - # won't be imported from scratch in each subprocess. - import stbt # pylint:disable=unused-variable - - selftest_dir = "%s/selftest" % os.curdir - mkdir_p(selftest_dir) - # We use this process pool for sandboxing rather than concurrency: - pool = multiprocessing.Pool( - processes=1, maxtasksperchild=1, initializer=init_worker) - tmpdir = tempfile.mkdtemp(dir=selftest_dir, prefix="auto_selftest") - modules = [] - try: - if not source_files: - source_files = valid_source_files(_recursive_glob('*.py')) - perf_log = [] - test_file_count = 0 - for module_filename in iterate_with_progress(source_files): - outname = os.path.join(tmpdir, selftest_filename(module_filename)) - barename = re.sub('.py$', '_bare.py', outname) - mkdir_p(os.path.dirname(outname)) - - module = pool.apply(inspect_module, (module_filename,)) - modules.append(module) - test_line_count = write_bare_doctest(module, barename) - if test_line_count: - test_file_count += 1 - perf_log.extend(pool.apply_async( - update_doctests, (barename, outname)).get(timeout=60 * 60)) - os.unlink(barename) - - if test_file_count > 0: - with open('%s/README' % tmpdir, 'w') as f: - f.write("\n".join(wrap( - "This directory contains self-tests generated by `stbt " - "auto-selftest`. Do not modify by hand. Any files " - "modified or created in this directory may be overwritten " - "or deleted by `stbt auto-selftest`.")) + "\n") - - for x in _recursive_glob('*.pyc', tmpdir): - os.unlink(os.path.join(tmpdir, x)) - prune_empty_directories(tmpdir) - - print_perf_summary(perf_log, time.time() - start_time) - - return tmpdir, modules - except: - pool.terminate() - pool.join() - shutil.rmtree(tmpdir) - raise - - -def valid_source_files(source_files): - filenames = [] - for module_filename in source_files: - if module_filename.startswith('selftest'): - continue - if not is_valid_python_identifier( - os.path.basename(module_filename)[:-3]): - continue - if not os.path.exists(module_filename): - continue - filenames.append(module_filename) - return filenames - - -def selftest_filename(module_filename): - return re.sub('.py$', '_selftest.py', module_filename) - - -class Module(namedtuple('Module', "filename items error")): - pass - - -class Item(namedtuple('Item', 'name expressions screenshots try_screenshots')): - pass - - -def inspect_module(module_filename): - """ - Pulls the relevant information from the module required to generate tests. - This is a seperate function so we can run it in a subprocess to avoid - contaminating the main processes. - """ - try: - out = [] - module = _import_by_filename(module_filename) - for x in dir(module): - item = getattr(module, x) - if getattr(item, '__module__', None) != module.__name__: - continue - expressions = list(getattr(item, 'AUTO_SELFTEST_EXPRESSIONS', [])) - if not expressions: - continue - - out.append(Item( - name=item.__name__, - expressions=expressions, - screenshots=list( - getattr(item, 'AUTO_SELFTEST_SCREENSHOTS', [])), - try_screenshots=list( - getattr(item, 'AUTO_SELFTEST_TRY_SCREENSHOTS', ['*.png'])))) - return Module(module_filename, out, None) - except (KeyboardInterrupt, SystemExit): - raise - except: # pylint: disable=bare-except - error_message = sys.exc_info()[1] - sys.stderr.write( - "Received \"%s\" exception while inspecting %s; skipping.\n" - % (error_message, module_filename)) - return Module(module_filename, [], error_message) - - -def write_bare_doctest(module, output_filename): - total_tests_written = 0 - - outfile = cStringIO.StringIO() - screenshots_rel = os.path.relpath( - SCREENSHOTS_ROOT, os.path.dirname(output_filename)) - - import_path, import_name = find_import_name(module.filename) - module_rel = os.path.relpath( - import_path, os.path.dirname(output_filename)) - - outfile.write(dedent(r''' #!/usr/bin/env python - # coding=utf-8 - """ - This file contains regression tests automatically generated by - ``stbt auto-selftest``. These tests are intended to capture the - behaviour of Frame Objects (and other helper functions that operate on - a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` - whenever you make a change to your Frame Objects, and use ``git diff`` - to see how your changes affect the behaviour of the Frame Object. - - NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- - it merely documents the behaviour at the time that - ``stbt auto-selftest`` was run. - """ - # pylint: disable=line-too-long - - import os - import sys - - sys.path.insert(0, os.path.join( - os.path.dirname(__file__), {module_rel})) - - from {name} import * # isort:skip pylint: disable=wildcard-import, import-error - - _FRAME_CACHE = {{}} - - - def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - {screenshots_rel}, name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - '''.format(name=import_name, - screenshots_rel=repr(screenshots_rel), - module_rel=repr(module_rel)))) - - for x in module.items: - total_tests_written += write_test_for_class(x, outfile) - if total_tests_written > 0: - with open(output_filename, 'w') as f: - f.write(outfile.getvalue()) - return total_tests_written - - -def write_test_for_class(item, out): - all_screenshots = _recursive_glob('*.png', SCREENSHOTS_ROOT) - - always_screenshots = [] - try_screenshots = [] - - for filename in all_screenshots: - if any(fnmatch.fnmatch(filename, x) for x in item.screenshots): - always_screenshots.append(filename) - elif any(fnmatch.fnmatch(filename, x) for x in item.try_screenshots): - try_screenshots.append(filename) - - if len(always_screenshots) + len(try_screenshots) == 0: - return 0 - - always_screenshots.sort() - try_screenshots.sort() - out.write(dedent('''\ - - - def auto_selftest_{name}(): - r""" - ''').format(name=item.name)) - - for expr in item.expressions: - for s in always_screenshots: - out.write(" >>> %s\n" % expr.format(frame='f("%s")' % s)) - for s in try_screenshots: - out.write(" >>> %s # remove-if-false\n" % expr.format( - frame='f("%s")' % s)) - - out.write(' """\n pass\n') - return len(always_screenshots) + len(try_screenshots) - - -def print_perf_summary(perf_log, total_time): - perf_log.sort(key=lambda x: -x[1]) - eval_time = sum(x[1] for x in perf_log) - print "Total time: %fs" % total_time - print "Total time evaluating: %fs" % eval_time - print "Overhead: %fs (%f%%)" % ( - total_time - eval_time, 100 * (total_time - eval_time) / total_time) - print "Number of expressions evaluated: %i" % len(perf_log) - if len(perf_log) == 0: - return - print "Median time: %fs" % perf_log[len(perf_log) // 2][1] - print "Slowest 10 evaluations:" - for cmd, duration in perf_log[:10]: - print "%.03fs\t%s" % (duration, cmd) - - -def update_doctests(infilename, outfile): - """ - Updates a file with doctests in it but no results to have "correct" results. - """ - module = _import_by_filename(infilename) - if isinstance(outfile, str): - outfile = open(outfile, 'w') - - perf_log = [] - - with open(infilename, 'r') as infile, cache(): - for line in infile: - # pylint: disable=cell-var-from-loop - m = re.match(r'\s*>>> (.*)\n', line) - if m: - cmd = m.group(1) - else: - outfile.write(("%s" % line).encode('utf-8')) - continue - - if line.endswith(' # remove-if-false\n'): - line = line.replace(' # remove-if-false\n', '\n') - remove_if_false = True - else: - remove_if_false = False - - # At the end of this result[0] will either be "statement", "falsey", - # "truey" or "exception": - result = ["statement"] - - # This lets us know if anything was printed excluding the - # value/exception produced by running the code. It counts toward - # our measure of interestingness - did_print = [None] - - # doctest can't cope with printing unicode strings with unicode - # codepoints in them. This detects if we're going to fall into this - # trap. - would_unicode_fail = [False] - - oldstdout = sys.stdout - io = StringIO.StringIO() - real_write = io.write - - def io_write(text, *args, **kwargs): - if isinstance(text, unicode): - try: - text = text.encode('ascii') - except UnicodeEncodeError: - would_unicode_fail[0] = True - text = text.encode('ascii', 'backslashreplace') - return real_write(text, *args, **kwargs) - - io.write = io_write - - def displayhook(value): - result[0] = "truthy" if bool(value) else "falsey" - did_print[0] = (io.tell() != 0) - if value is not None: - print repr(value) - - try: - start_time = time.time() - sys.stdout = io - old_displayhook, sys.displayhook = sys.displayhook, displayhook - exec compile(cmd, "", "single") in module.__dict__ # pylint: disable=exec-used - if did_print[0] is None: - did_print[0] = (io.tell() != 0) - except Exception: # pylint: disable=broad-except - did_print[0] = (io.tell() != 0) - result[0] = "exception" - traceback.print_exc(0, io) - finally: - perf_log.append((cmd, time.time() - start_time)) - - interesting = (did_print[0] or - result[0] in ["exception", "truthy"]) - sys.displayhook = old_displayhook - sys.stdout = oldstdout - if interesting or not remove_if_false: - if would_unicode_fail[0]: - line = re.sub(r"\n$", " # doctest: +SKIP\n", line) - outfile.write("%s" % line) - io.seek(0) - for output_line in io: - if output_line.strip() == '': - outfile.write(' \n') - else: - outfile.write(' ' + output_line) - - return perf_log - - -def test_update_doctests(): - # We test that what we generate is what we expect. make check-pytest - # will check that the generated doctest passes as a doctest itself. - import subprocess - from tempfile import NamedTemporaryFile - with NamedTemporaryFile() as outfile: - test_line_count = len(update_doctests( - _find_file('tests/auto_selftest_bare.py'), outfile.name)) - assert test_line_count == 19 - actual = outfile.read() - with open(_find_file('tests/auto_selftest_expected.py')) as f: - expected = f.read() - if actual != expected: - subprocess.call( - ['diff', '-u', _find_file('tests/auto_selftest_expected.py'), - outfile.name]) - assert actual == expected - - -def _recursive_glob(expr, dir_=None): - if dir_ is None: - dir_ = os.curdir - matches = [] - for root, _, filenames in os.walk(dir_): - for filename in fnmatch.filter(filenames, expr): - matches.append(os.path.relpath(os.path.join(root, filename), dir_)) - return matches - - -def _find_file(path, root=os.path.dirname(os.path.abspath(__file__))): - return os.path.join(root, path) - - -def _find_test_pack_root(): - root = os.getcwd() - while root != '/': - if os.path.exists(os.path.join(root, '.stbt.conf')): - return root - root = os.path.split(root)[0] - - -def _import_by_filename(filename): - from importlib import import_module - import_dir, import_name = find_import_name(filename) - sys.path.insert(0, import_dir) - return import_module(import_name) - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-batch stb-tester-31/stbt-batch --- stb-tester-30-5-gbefe47c/stbt-batch 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch 1970-01-01 00:00:00.000000000 +0000 @@ -1,31 +0,0 @@ -#!/bin/sh - -# Copyright 2013 YouView TV Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -#/ usage: stbt batch [args] -#/ -#/ Available subcommands are: -#/ run Run the specified stbt script[s], create report -#/ report Re-generate the report for previous test runs -#/ instaweb Start a web server to view & edit the report -#/ -#/ For help on a specific subcommand do 'stbt batch -h'. - -usage() { grep '^#/' "$0" | cut -c4-; } - -[ $# -ge 1 ] || { usage >&2; exit 1; } -cmd="$1" -shift -case "$cmd" in - -h|--help) - usage; exit 0;; - run) - PYTHONPATH=$(dirname "$0"):$PYTHONPATH \ - exec "$(dirname "$0")"/stbt-batch.d/run.py "$@";; - report|instaweb) - exec "$(dirname "$0")"/stbt-batch.d/$cmd "$@";; - *) - usage >&2; exit 1;; -esac diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/instaweb stb-tester-31/stbt-batch.d/instaweb --- stb-tester-30-5-gbefe47c/stbt-batch.d/instaweb 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/instaweb 1970-01-01 00:00:00.000000000 +0000 @@ -1,111 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2013 YouView TV Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -# HTTP server that serves HTML reports on the results from tests run with `stbt -# batch run`. Allows interactive editing of testrun failure-reason and notes. -# Uses Flask, a python web micro-framework: http://flask.pocoo.org -# Run with `cd results-directory; stbt batch instaweb 0.0.0.0:8080` - -import argparse -import mimetypes -import os -import subprocess -import sys -from os.path import abspath, basename, dirname, isdir, isfile, normpath - -from flask import (abort, Flask, redirect, render_template, request, - send_from_directory) - -app = Flask(__name__, instance_path=os.getcwd()) - -mimetypes.init() -mimetypes.types_map[".log"] = "text/plain" -mimetypes.types_map[".webm"] = "video/webm" - - -@app.route("/", defaults={"path": ""}) -@app.route("/") -def catch_all(path): - f = normpath(os.path.join(app.instance_path, path)) - if not f.startswith(app.instance_path): - abort(404) - - if isfile(f): - return send_from_directory(app.instance_path, path, cache_timeout=1) - if isdir(f): - if not path.endswith("/") and path != "": - return redirect(path + "/") - index = os.path.join(path, "index.html") - if isfile(index): - return send_from_directory( - app.instance_path, index, cache_timeout=1) - else: - return render_template( - "directory-index.html", - dir=(path or "/"), - entries=[ - x + "/" if isdir(os.path.join(f, x)) else x - for x in sorted(os.listdir(f))]) - - abort(404) - - -@app.route( - "//failure-reason", methods=["POST"], - defaults={"filename": "failure-reason"}) -@app.route( - "//notes", methods=["POST"], defaults={"filename": "notes"}) -def update_file(testrun, filename): - d = testrun_directory(testrun) - manual_text = request.form["value"].encode('utf-8', errors="ignore").strip() - f = os.path.join(d, filename) - automatic_text = open(f).read().strip() if isfile(f) else "" - if len(manual_text) == 0 or manual_text == automatic_text: - try: - os.remove(f + ".manual") - except OSError: - pass - else: - with open(f + ".manual", "w") as ff: - ff.write(manual_text) - subprocess.Popen([ - os.path.join(dirname(abspath(__file__)), "report"), - "--html-only", - d]) - return manual_text if len(manual_text) > 0 else automatic_text - - -@app.route("//delete", methods=["POST"]) -def hide_testrun(testrun): - d = testrun_directory(testrun) - os.rename(d, os.path.join(dirname(d), "." + basename(d))) - subprocess.Popen([ - os.path.join(dirname(abspath(__file__)), "report"), - "--html-only", - os.path.join(dirname(d), "index.html")]) - return "" - - -def testrun_directory(path): - d = normpath(os.path.join(app.instance_path, path)) - if (not d.startswith(app.instance_path) or not isdir(d) or - not isfile(os.path.join(d, "test-name"))): - abort(404) - return d - - -if __name__ == "__main__": - p = argparse.ArgumentParser() - p.prog = "stbt batch instaweb" - p.add_argument( - "--debug", action="store_true", - help="Enable python backtraces and application reloading") - p.add_argument( - "listen", metavar="address:port", default="localhost:5000", nargs="?", - help="The address and port to listen on (default: %(default)s)") - args = p.parse_args(sys.argv[1:]) - host, port = args.listen.split(":") - app.run(host=host, port=int(port), debug=args.debug) diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/post-run.sh stb-tester-31/stbt-batch.d/post-run.sh --- stb-tester-30-5-gbefe47c/stbt-batch.d/post-run.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/post-run.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# -*- sh-basic-offset: 2 -*- - -# Copyright 2013 YouView TV Ltd. -# 2013-2015 stb-tester.com Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). -# -# Input environment variables: -# -# * $stbt_root -# -# Outputs: -# -# * Files in the current working directory -# - -main() { - grep -q "FAIL: .*: MatchTimeout" stdout.log && template - grep -q "FAIL: .*: NoVideo" stdout.log && { - check_capture_hardware || touch unrecoverable-error; } -} - -template() { - local template=$( - sed -n 's,^.*stbt-run: Searching for $.*\.png$$,\1,p' stderr.log | - tail -1) - [ -f "$template" ] && cp "$template" template.png -} - -check_capture_hardware() { - case "$("$stbt_root"/stbt-config global.source_pipeline | awk '{print $1}')" in - v4l2src) - if grep -q "Cannot identify device '/dev/v" failure-reason; then - echo "v4l2 device not found; exiting." - return 1 - fi - ;; - - decklinksrc) - ( echo "$(basename "$0"): Checking Blackmagic video-capture device" - GST_DEBUG=decklinksrc:5 GST_DEBUG_NO_COLOR=1 \ - "$stbt_root"/stbt-run --sink-pipeline='' \ - <(echo "import time; time.sleep(1)") 2>&1 - ) | ts '[%Y-%m-%d %H:%M:%.S %z] ' > decklinksrc.log - - if grep -q "enable video input failed" decklinksrc.log; then - local subdevice=$( - "$stbt_root"/stbt-config global.source_pipeline | - grep -o device-number=. | awk -F= '{print $2}') - local users=$( - lsof -F Lnc \ - /dev/blackmagic${subdevice:-0} \ - /dev/blackmagic/dv${subdevice:-0} \ - 2>/dev/null | - # Example `lsof` output: - # p70752 - # cgst-launch-0.10 - # Lstb-tester - # n/dev/blackmagic0 - awk '/^p/ { printf "\n" } - { sub(/^./, ""); printf $0 " " }') - echo "Blackmagic card in use: $users" > failure-reason - cp failure-reason failure-reason.manual - echo "Blackmagic card in use; exiting." - return 1 - - # Even if the card has no video connected to its input you see - # "VideoInputFrameArrived: Frame received - No input signal detected" - elif ! grep -q VideoInputFrameArrived decklinksrc.log; then - echo "Blackmagic card froze" > failure-reason - cp failure-reason failure-reason.manual - echo "Blackmagic card froze; exiting." - return 1 - fi - ;; - esac -} - -trap on_term sigterm -on_term() { - # Ignore SIGTERM. It will have been sent to the whole process group, but we - # want this process to finish running to write out the right results files. - true; -} - -main "$@" diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/print_backtrace.gdb stb-tester-31/stbt-batch.d/print_backtrace.gdb --- stb-tester-30-5-gbefe47c/stbt-batch.d/print_backtrace.gdb 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/print_backtrace.gdb 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -thread apply all bt diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/README.rst stb-tester-31/stbt-batch.d/README.rst --- stb-tester-30-5-gbefe47c/stbt-batch.d/README.rst 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/README.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,129 +0,0 @@ -Test runner & reporting system -============================== - -run ---- - -This:: - - stbt batch run path/to/test.py another/test.py - -will run the given stb-tester scripts until one of them fails. You can run the -tests once, or keep going after uninteresting failures, or keep going no matter -what; see ``run -h`` for help. - -``stbt batch run`` creates a separate timestamped directory for each test run, -containing the logs from that run. - -report ------- - -After each test run, ``stbt batch run`` executes ``stbt batch report`` to -classify the failure reason, gather other useful information, and generate a -static html report in ``index.html``. So the way you should use ``stbt batch -run`` is something like this:: - - mkdir my-test-session - cd my-test-session - stbt batch run path/to/test.py & - firefox index.html - -It's up to you to organise "test sessions" as you want them: You can run -further tests from the same directory to add them to the existing report, -or run them from a new directory for a separate report. - -``stbt batch run`` executes ``stbt batch report`` automatically, but you can -also re-run ``stbt batch report`` on old test logs (when you've added new -classifications after those tests were originally run; see the "classify" user -hook below). See ``stbt batch report -h`` for help. - -User hooks ----------- - -The following hooks are available for custom log collection, failure -classification, and failure recovery. Each hook is a variable in the stbt -configuration file; if the variable is set to an executable program, ``run`` -will invoke that program at the appropriate time, with the current working -directory set to the directory containing the test run logs: - -**batch.pre_run** - Invoked immediately before the test is run, with the - single command-line argument "start". Intended for starting custom logging - processes. - -**batch.post_run** - Invoked as soon as possible after the test has completed, with the single - command-line argument "stop" (so that you can set ``pre_run`` and - ``post_run`` to the same program). Intended for stopping custom logging - processes and gathering any other data that must be gathered immediately - after the test has run. - - The program should save all logs to the current working directory. This - program should not do any expensive analysis that could be done later. - - Communication between the ``pre_run`` and the ``post_run`` programs - can be achieved by writing files to the current working directory (for - example pid files), which the ``post_run`` program should clean up. - -**batch.recover** - Invoked after the test has failed. This program should restore the - device-under-test to a state where it is ready to run the next test (for - example by power-cycling the device-under-test and ensuring the boot sequence - completes). This program is invoked after the built-in classification and the - custom ``classify`` program, so the output from that classification is - available (see below). - - This program should return a non-zero exit status to indicate that recovery - was unsuccessful and no further tests should be run. - -**batch.classify** - Invoked after the test has completed (after the ``post_run`` program and - after the built-in classification / log analysis). Intended for additional - analysis of the log files. This program is also invoked when ``report`` is - run by the user, so this program shouldn't rely on any information from the - live test-run environment. - - The current working directory will contain the following files: - - * ``test-name`` (containing the path to the test script). - * ``exit-status`` (containing the numeric exit status from ``stbt run``). - * ``failure-reason`` (containing the failure reason determined by the - built-in classification -- see ``report`` for details). - * ``duration`` (containing the test-run duration in seconds). - * ``stdout.log``, ``stderr.log`` (containing the output from ``stbt run``). - * ``sensors.log`` (if ``lm-sensors`` is installed; contains hardware sensor - readings such as CPU temperature). - * ``backtrace.log`` (if ``stbt run`` dumped core). - * ``screenshot-clean.png`` (taken as soon as the test failed). - * ``template.png`` (if the test failed due to a MatchTimeout; contains the - image that the ``wait_for_match`` function failed to find). - - The program can choose to leave the ``failure-reason`` file unchanged, or to - overwrite the file with a new failure reason. The failure reason is an - arbitrary string that is shown in the "Exit status" column of the html - report. - - The program can add additional columns to the html report by writing to the - file ``extra-columns``. Each line should contain a key (the column header), - followed by a tab, followed by a value (arbitrary text). Multiple lines with - the same key will have their values merged into a single column. The program - should append to the ``extra-columns`` file, not overwrite it. - -instaweb --------- - -The generated report is a set of static html files, which you can view locally -(using a `file:///...` url), or you can serve them with a web server like -apache. But if you want to interactively *edit* the report, you can run ``stbt -batch instaweb``. By default, this serves on ``localhost:5000``. To serve on -all public network interfaces, run it like this:: - - stbt batch instaweb 0.0.0.0:5000 - -Run ``stbt batch instaweb`` from the directory containing the test results (for -example ``my-test-session`` in the "report" section above). - -``stbt batch instaweb`` probably can't handle high loads or many concurrent -users. For such cases you should proxy ``stbt batch instaweb`` behind Apache or -Nginx. ``stbt batch instaweb`` uses "Flask", a python web micro-framework; for -deployment options see http://flask.pocoo.org/docs/deploying/ diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/report stb-tester-31/stbt-batch.d/report --- stb-tester-30-5-gbefe47c/stbt-batch.d/report 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/report 1970-01-01 00:00:00.000000000 +0000 @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -*- sh-basic-offset: 2 -*- - -# Copyright 2013 YouView TV Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -#/ Usage: stbt batch report [...] -#/ -#/ Classifies the results previously generated by 'stbt batch run' -#/ according to the reason for failure. -#/ Generates a detailed report in /index.html -#/ and a summary report in /../index.html. - -IFS=$'\n' - -usage() { grep '^#/' "$0" | cut -c4-; } - -main() { - runner=$(dirname "$(realpath "$0")") - - do_classify=true - do_html=true - while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) usage; exit 0;; - --classify-only) do_html=false; shift;; - --html-only) do_classify=false; shift;; - *) break;; - esac - done - [[ $# -gt 0 ]] || { usage >&2; exit 1; } - - for rundir in "$@"; do - ( - [[ "$rundir" =~ index.html$ ]] && return 0 # Summary report only - cd "$rundir" || die "Invalid directory '$rundir'" - - if $do_classify; then - [ -r exit-status ] || { - echo "$rundir: No exit status (test still in progress); skipping" - return 0 - } - classify > failure-reason - LC_ALL=C sort --merge stdout.log stderr.log > combined.log - user_command classify - echo "$rundir: $(cat failure-reason)" - fi - - if $do_html; then - [ -r test-name ] || { - echo "$rundir: Not a test result directory; skipping" - return 0 - } - python "$runner"/report.py . > index.html.$$ && - mv index.html.$$ index.html - fi - ) || return - done - - # Generate summary report - if $do_html; then - for resultsdir in $( - for rundir in "$@"; do dirname "$(abspath "$rundir")"; done | sort | uniq) - do - ( - cd "$resultsdir" || die "Invalid directory '$resultsdir'" - python "$runner"/report.py index.html > index.html.$$ && - mv index.html.$$ index.html - ) - done - fi -} - -classify() { - local status=$(cat exit-status) - local testname=$(cat test-name) - - if [ $status -eq 0 ]; then - echo success - - elif [ $status -gt 128 ]; then - echo killed "($(signalname $((status - 128))))" - - elif local exception=$( - sed -n "s/^.*FAIL: .*$(basename "$testname"): //p" stdout.log); - [[ -n "$exception" ]]; - then - echo "$exception" - - else - echo unknown - fi -} - -signalname() { - local num=$1 - ( set -o pipefail - kill -l | grep -Eo "\b$num\) SIG\S+" | awk '{print tolower($2)}' || - echo $num - ) -} - -user_command() { - local c=$("$runner"/../stbt-config batch.$1 2>/dev/null) - [[ -z "$c" ]] && return - "$c" -} - -die() { echo "$(basename "$0"): error: $*" >&2; exit 1; } - -# Portable implementation of GNU "readlink -f" to support BSD/OSX. -realpath() { - python -c 'import os, sys; print os.path.realpath(sys.argv[1])' "$1" -} - -abspath() { - python -c 'import os, sys; print os.path.abspath(sys.argv[1])' "$1" -} - -main "$@" diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/report.py stb-tester-31/stbt-batch.d/report.py --- stb-tester-30-5-gbefe47c/stbt-batch.d/report.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/report.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,161 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2013 YouView TV Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -"""Generates reports from logs of stb-tester test runs created by 'run'.""" - -import collections -import glob -import itertools -import os -import re -import sys -from datetime import datetime -from os.path import abspath, basename, dirname, isdir - -import jinja2 - -escape = jinja2.Markup.escape - - -templates = jinja2.Environment(loader=jinja2.FileSystemLoader( - os.path.join(os.path.dirname(__file__), "templates"))) - - -def main(argv): - usage = "Usage: report (index.html | )" - if len(argv[1:]) == 0: - die(usage) - if argv[1] in ("-h", "--help"): - print usage - sys.exit(0) - for target in argv[1:]: - if isdir(target): - match = re.match( - r"(.*/)?\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}(-[^/]+)?$", - abspath(target)) - if match: - testrun(match.group()) - elif target.endswith("index.html"): - index(dirname(target)) - else: - die("Invalid target '%s'" % target) - - -def index(parentdir): - rundirs = [ - dirname(x) for x in glob.glob( - os.path.join(parentdir, "????-??-??_??.??.??*/test-name"))] - runs = [Run(d) for d in sorted(rundirs, reverse=True)] - if len(runs) == 0: - die("Directory '%s' doesn't contain any testruns" % parentdir) - print templates.get_template("index.html").render( # pylint:disable=no-member - name=basename(abspath(parentdir)).replace("_", " "), - runs=runs, - extra_columns=set( - itertools.chain(*[x.extra_columns.keys() for x in runs])), - ).encode('utf-8') - - -def testrun(rundir): - print templates.get_template("testrun.html").render( # pylint:disable=no-member - run=Run(rundir), - ).encode('utf-8') - - -class Run(object): - def __init__(self, rundir): - self.rundir = rundir - - try: - self.exit_status = int(self.read("exit-status")) - except ValueError: - self.exit_status = "still running" # pylint:disable=redefined-variable-type - - self.duration = self.read_seconds("duration") - self.failure_reason = self.read("failure-reason") - self.git_commit = self.read("git-commit") - self.notes = self.read("notes") - self.test_args = self.read("test-args") - self.test_name = self.read("test-name") - - if os.path.exists(rundir + '/video.webm'): - self.video = 'video.webm' - - if self.exit_status != "still running": - self.files = sorted([ - basename(x) for x in glob.glob(rundir + "/*") - if basename(x) not in [ - "duration", - "exit-status", - "extra-columns", - "failure-reason", - "git-commit", - "test-args", - "test-name", - ] and - not x.endswith(".jpeg") and - not x.endswith(".jpg") and - not x.endswith(".png") and - not x.endswith(".manual") and - not basename(x).startswith("index.html") - ]) - self.images = sorted( - set([ - basename(x) for x in - glob.glob(rundir + "/*.jpeg") + - glob.glob(rundir + "/*.jpg") + - glob.glob(rundir + "/*.png")]) - - set(["thumbnail.jpg"])) - - self.extra_columns = collections.OrderedDict() - for line in self.read("extra-columns").splitlines(): - try: - column, value = line.split("\t", 1) - self.extra_columns.setdefault(column.strip(), []) - self.extra_columns[column.strip()].append(value.strip()) - except ValueError: - pass - - t = re.match( - r"\d{4}-\d{2}-\d{2}_\d{2}\.\d{2}\.\d{2}", basename(rundir)) - assert t, "Invalid rundir '%s'" % rundir - self.timestamp = datetime.strptime(t.group(), "%Y-%m-%d_%H.%M.%S") - - def css_class(self): - if self.exit_status == "still running": - return "muted" # White - elif self.exit_status == 0: - return "success" - elif self.exit_status == 1: - return "error" # Red: Possible device-under-test failure - else: - return "warning" # Yellow: Test infrastructure error - - def read(self, f): - f = os.path.join(self.rundir, f) - if os.path.exists(f + ".manual"): - return escape(open(f + ".manual").read().decode('utf-8').strip()) - elif os.path.exists(f): - return open(f).read().decode('utf-8').strip() - else: - return "" - - def read_seconds(self, f): - s = self.read(f) - try: - s = int(s) - except ValueError: - s = 0 - return "%02d:%02d:%02d" % (s / 3600, (s % 3600) / 60, s % 60) - - -def die(message): - sys.stderr.write("report.py: %s\n" % message) - sys.exit(1) - - -if __name__ == "__main__": - main(sys.argv) diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/run.py stb-tester-31/stbt-batch.d/run.py --- stb-tester-30-5-gbefe47c/stbt-batch.d/run.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/run.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,605 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015-2018 stb-tester.com Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - - -import argparse -import datetime -import errno -import glob -import os -import random -import signal -import subprocess -import sys -import threading -import time -from collections import namedtuple -from contextlib import contextmanager -from distutils.spawn import find_executable - -from _stbt.utils import mkdir_p - - -def main(argv): - parser = argparse.ArgumentParser(usage=( - "\n stbt batch run [options] test.py [test.py ...]" - "\n stbt batch run [options] test.py arg [arg ...] -- " - "test.py arg [arg ...] [-- ...])")) - parser.add_argument( - '-1', '--run-once', action="store_true", help=( - 'Run once. The default behaviour is to run the test repeatedly as ' - 'long as it passes.')) - parser.add_argument( - '-k', '--keep-going', action="count", help=( - 'Continue running after failures. Provide this argument once to ' - 'continue running after "uninteresting" failures, and twice to ' - 'continue running after any failure (except those that would ' - 'prevent any further test from passing).')) - parser.add_argument( - '-d', '--debug', action="store_true", help=( - 'Enable "stbt-debug" dump of intermediate images.')) - parser.add_argument( - '-v', '--verbose', action="count", default=0, help=( - 'Verbose. Provide this argument once to print stbt standard ' - 'output. Provide this argument twice to also print stbt stderr ' - 'output.')) - parser.add_argument( - '-o', '--output', default=os.curdir, help=( - 'Output directory to save the report and test-run logs under ' - '(defaults to the current directory).')) - parser.add_argument( - '-t', '--tag', help=( - 'Tag to add to test-run directory names (useful to differentiate ' - 'directories when you intend to merge test results from multiple ' - 'machines).')) - parser.add_argument( - '--shuffle', action="store_true", help=( - "Run the test cases in a random order attempting to spend the same " - "total amount of time executing each test case.")) - parser.add_argument( - '--no-html-report', action='store_false', dest='do_html_report', - help="""Don't generate an HTML report after each test-run; generating - the report can be slow if there are many results in the output - directory. You can still generate the HTML reports afterwards with - 'stbt batch report'.""") - parser.add_argument( - '--no-save-video', action='store_false', dest='do_save_video', help=""" - Don't generate a video recording of each test-run. Use this if you - are saving video another way.""") - parser.add_argument('test_name', nargs=argparse.REMAINDER) - args = parser.parse_args(argv[1:]) - - if args.tag is not None: - tag_suffix = '-' + args.tag - else: - tag_suffix = "" - - os.environ['PYTHONUNBUFFERED'] = 'x' - - term_count = [0] - - def on_term(_signo, _frame): - term_count[0] += 1 - if term_count[0] == 1: - sys.stderr.write( - "\nReceived interrupt; waiting for current test to complete.\n") - else: - sys.stderr.write("Received interrupt; exiting.\n") - sys.exit(1) - - signal.signal(signal.SIGINT, on_term) - signal.signal(signal.SIGTERM, on_term) - - failure_count = 0 - last_exit_status = 0 - - test_cases = parse_test_args(args.test_name) - - run_count = 0 - - if args.shuffle: - test_generator = shuffle(test_cases, repeat=not args.run_once) - else: - test_generator = loop_tests(test_cases, repeat=not args.run_once) - - mkdir_p(args.output) - - # We assume that all test-cases are in the same git repo: - git_info = read_git_info(os.path.dirname(test_cases[0][0])) - - for test_name, test_args in test_generator: - if term_count[0] > 0: - break - run_count += 1 - - last_exit_status = run_test( - args, tag_suffix, test_name, test_args, git_info) - - if last_exit_status != 0: - failure_count += 1 - if os.path.exists( - "%s/latest%s/unrecoverable-error" % (args.output, tag_suffix)): - break - - if last_exit_status == 0: - continue - elif last_exit_status >= 2 and args.keep_going > 0: - # "Uninteresting" failures due to the test infrastructure - continue - elif args.keep_going >= 2: - continue - else: - break - - if run_count == 1: - # If we only run a single test a single time propagate the result - # through - return last_exit_status - elif failure_count == 0: - return 0 - else: - return 1 - - -@contextmanager -def html_report(batch_args, rundir): - if batch_args.do_html_report: - try: - subprocess.call([_find_file("report"), '--html-only', rundir], - stdout=DEVNULL_W) - yield - finally: - subprocess.call([_find_file("report"), '--html-only', rundir], - stdout=DEVNULL_W) - else: - yield - - -def stdout_logging(batch_args, test_name, test_args): - display_name = " ".join((test_name,) + test_args) - if batch_args.verbose > 0: - print "\n%s ..." % display_name - else: - print "%s ... " % display_name, - - -def post_run_stdout_logging(exit_status): - if exit_status == 0: - print "OK" - else: - print "FAILED" - - -@contextmanager -def record_duration(rundir): - start_time = time.time() - try: - yield - finally: - with open("%s/duration" % rundir, 'w') as f: - f.write(str(time.time() - start_time)) - - -def run_test(batch_args, tag_suffix, test_name, test_args, - git_info): - with setup_dirs(batch_args.output, tag_suffix) as rundir: - fill_in_data_files(rundir, test_name, test_args, git_info, - batch_args.tag) - with html_report(batch_args, rundir): - user_command("pre_run", ["start"], cwd=rundir) - stdout_logging(batch_args, test_name, test_args) - with record_duration(rundir): - exit_status = run_stbt_run( - test_name, test_args, batch_args, cwd=rundir) - - # stbt batch run used to be written in bash - so it expects - # exit_status to follow the conventions of $?. If the child - # died due to a signal the value should be 128 + signo rather - # than Python's -signo: - if exit_status < 0: - exit_status = 128 - exit_status - - post_run_stdout_logging(exit_status) - with open("%s/exit-status" % rundir, 'w') as f: - f.write("%i" % exit_status) - - collect_sensors_data(rundir) - user_command("post_run", ["stop"], cwd=rundir) - with open("%s/failure-reason" % rundir, 'wb') as f: - f.write(get_failure_reason(test_name, exit_status, cwd=rundir) + - '\n') - - corefiles_to_backtraces(rundir) - - post_run_script(exit_status, rundir) - - combine_logs(rundir) - - user_command("classify", [], cwd=rundir) - - if exit_status != 0: - if user_command("recover", [], cwd=rundir) != 0: - with open("%s/unrecoverable-error" % rundir, 'w'): - pass - - return exit_status - - -GitInfo = namedtuple('GitInfo', 'commit commit_sha git_dir') - - -def read_git_info(testdir): - try: - def git(*cmd): - return subprocess.check_output(('git',) + cmd, cwd=testdir).strip() - return GitInfo( - git('describe', '--always', '--dirty'), - git('rev-parse', 'HEAD'), - git('rev-parse', '--show-toplevel')) - except subprocess.CalledProcessError: - return None - except OSError as e: - # ENOENT means that git is not in $PATH - if e.errno != errno.ENOENT: - raise - - -def make_rundir(outputdir, tag): - for n in range(2): - rundir = datetime.datetime.now().strftime("%Y-%m-%d_%H.%M.%S") + tag - try: - os.mkdir(os.path.join(outputdir, rundir)) - return rundir - except OSError as e: - if e.errno == errno.EEXIST and n == 0: - # Avoid directory name clashes if the previous test took <1s to - # run - time.sleep(1) - else: - raise - - -def fill_in_data_files(rundir, test_name, test_args, git_info, tag): - def write_file(name, data): - with open(os.path.join(rundir, name), 'w') as f: - f.write(data) - - if git_info: - write_file("git-commit", git_info.commit) - write_file("git-commit-sha", git_info.commit_sha) - write_file("test-name", os.path.relpath(test_name, git_info.git_dir)) - else: - write_file("test-name", os.path.abspath(test_name)) - write_file("test-args", "\n".join(test_args)) - - stbt_version = os.environ.get("STBT_VERSION") - if stbt_version: - write_file("stbt-version.log", stbt_version + "\n") - - if tag: - write_file("extra-columns", "Tag\t%s\n" % tag) - - -@contextmanager -def setup_dirs(outputdir, tag): - mkdir_p(outputdir) - - rundir = make_rundir(outputdir, tag) - - symlink_f(rundir, os.path.join(outputdir, "current" + tag)) - - try: - yield os.path.join(outputdir, rundir) - finally: - # Now test has finished... - symlink_f(rundir, os.path.join(outputdir, "latest" + tag)) - - -def symlink_f(source, link_name): - name = "%s-%06i~" % (link_name, random.randint(0, 999999)) - os.symlink(source, name) - os.rename(name, link_name) - - -DEVNULL_R = open('/dev/null') -DEVNULL_W = open('/dev/null', 'w') - - -def background_tee(pipe, filename, also_stdout=False): - def tee(): - with open(filename, 'w') as f: - while True: - line = pipe.readline() - if line == "": - # EOF - break - t = datetime.datetime.now() - line = t.strftime("[%Y-%m-%d %H:%M:%S.%f %z] ") + line - f.write(line) - if also_stdout: - sys.stdout.write(line) - sys.stdout.flush() - - t = threading.Thread(target=tee) - t.daemon = True - t.start() - return t - - -def run_stbt_run(test_name, test_args, batch_args, cwd): - """ - Invoke stbt run with the appropriate arguments, redirecting output to log - files. - """ - cmd = [_find_file('../stbt-run'), '--save-thumbnail=always'] - if batch_args.do_save_video: - cmd += ['--save-video=video.webm'] - if batch_args.debug: - cmd += ['-vv'] - else: - cmd += ['-v'] - cmd += [os.path.abspath(test_name), '--'] + list(test_args) - - child = None - t1 = t2 = None - child = subprocess.Popen( - cmd, stdin=DEVNULL_R, - preexec_fn=lambda: os.setpgid(0, 0), cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - t1 = background_tee( - child.stdout, "%s/stdout.log" % cwd, batch_args.verbose > 0) - t2 = background_tee( - child.stderr, "%s/stderr.log" % cwd, batch_args.verbose > 1) - return child.wait() - except SystemExit: - os.kill(-child.pid, signal.SIGTERM) - return child.wait() - finally: - if child.poll() is not None: - if t1: - t1.join() - if t2: - t2.join() - - -def collect_sensors_data(cwd): - try: - out = subprocess.check_output(['sensors']) - with open("%s/sensors.log" % cwd, 'w') as f: - f.write(out) - except OSError as e: - if e.errno != errno.ENOENT: - # sensors is not installed - raise - - -def post_run_script(exit_status, cwd): - """stbt-batch run used to be written in bash. We're porting to Python - incrementally. This function calls out to the bash implementation that - remains. - """ - subenv = dict(os.environ) - subenv['stbt_root'] = _find_file('..') - subenv['exit_status'] = str(exit_status) - subprocess.call( - [_find_file("post-run.sh")], stdin=DEVNULL_R, - env=subenv, preexec_fn=lambda: os.setpgid(0, 0), cwd=cwd) - - -def combine_logs(cwd): - # We should do this on the fly really - subenv = dict(os.environ) - subenv['LC_ALL'] = 'C' - with open("%s/combined.log" % cwd, "w") as f: - subprocess.check_call( - ['sort', '--merge', '%s/stdout.log' % cwd, - '%s/stderr.log' % cwd], env=subenv, stdout=f) - - -SIGNALS_TO_NAMES_DICT = { - getattr(signal, n): n - for n in dir(signal) - if n.startswith('SIG') and '_' not in n} - - -def get_failure_reason(test_name, exit_status, cwd): - if exit_status == 0: - return "success" - elif exit_status > 128: - return "killed (%s)" % SIGNALS_TO_NAMES_DICT[exit_status - 128].lower() - - exception = subprocess.check_output([ - "sed", "-n", "s/^.*FAIL: .*%s: //p" % os.path.basename(test_name), - "%s/stdout.log" % cwd]).strip() - if exception: - return exception - else: - return "unknown" - - -def user_command(name, args, cwd): - from _stbt.config import get_config - script = get_config("batch", name) - if script: - return subprocess.call([script] + args, stdin=DEVNULL_R, cwd=cwd) - else: - return 0 - - -def corefiles_to_backtraces(cwd): - for corefile in glob.glob("%s/core*" % cwd): - with open("%s/backtrace.log" % cwd, "w") as f: - try: - subprocess.call( - ['gdb', find_executable('python'), corefile, '-batch', - '-x', _find_file("print_backtrace.gdb")], - stdout=f, stderr=f) - except OSError as e: - if e.errno == errno.ENOENT: - sys.stderr.write( - "Failed to generate backtrace from core file %s: gdb " - "not installed\n" % corefile) - else: - raise - - -def listsplit(l, v): - """ - A bit like str.split, but for lists - - >>> listsplit(['test 1', '--', 'test 2', 'arg1', '--', 'test3'], '--') - [['test 1'], ['test 2', 'arg1'], ['test3']] - """ - out = [] - sublist = [] - for x in l: - if x == v: - if sublist: - out.append(sublist) - sublist = [] - else: - sublist.append(x) - if sublist: - out.append(sublist) - return out - - -def parse_test_args(args): - """ - >>> parse_test_args(['test 1.py', 'test2.py', 'test3.py']) - [('test 1.py', ()), ('test2.py', ()), ('test3.py', ())] - >>> parse_test_args(['test1.py', 'test2.py']) - [('test1.py', ()), ('test2.py', ())] - >>> parse_test_args(['test1.py', '--']) - [('test1.py', ())] - >>> parse_test_args(['test1.py', '--', 'test2.py']) - [('test1.py', ()), ('test2.py', ())] - >>> parse_test_args(['test1.py', '--', 'test2.py', '--']) - [('test1.py', ()), ('test2.py', ())] - >>> parse_test_args(['test1.py', 'test2.py']) - [('test1.py', ()), ('test2.py', ())] - >>> parse_test_args( - ... ['test1.py', 'arg1', 'arg2', '--', 'test2.py', 'arg', '--', - ... 'test3.py']) - [('test1.py', ('arg1', 'arg2')), ('test2.py', ('arg',)), ('test3.py', ())] - """ - if '--' in args: - return [(x[0], tuple(x[1:])) for x in listsplit(args, '--')] - else: - return [(x, ()) for x in args] - - -def loop_tests(test_cases, repeat=True): - while True: - for test in test_cases: - yield test - if not repeat: - return - - -def weighted_choice(choices): - """ - See http://stackoverflow.com/questions/3679694/ - """ - total = sum(w for c, w in choices) - r = random.uniform(0, total) - upto = 0 - for c, w in choices: - if upto + w > r: - return c - upto += w - assert False, "Shouldn't get here" - - -def shuffle(test_cases, repeat=True): - test_cases = test_cases[:] - random.shuffle(test_cases) - timings = {test: [0.0, 0] for test in test_cases} - - # Run all the tests first time round: - for test in test_cases: - start_time = time.time() - yield test - timings[test][0] += time.time() - start_time - timings[test][1] += 1 - - if not repeat: - return - - while True: - test = weighted_choice([(k, v[1] / v[0]) for k, v in timings.items()]) - start_time = time.time() - yield test - timings[test][0] += time.time() - start_time - timings[test][1] += 1 - - -def test_that_shuffle_runs_through_all_tests_initially_with_repeat(): - from itertools import islice - - test_cases = range(20) - out = list(islice(shuffle(test_cases), 20)) - - # They must be randomised: - assert test_cases != out - - # But all of them must have been run - assert test_cases == sorted(out) - - -def test_that_shuffle_runs_through_all_tests_no_repeat(): - test_cases = range(20) - out = list(shuffle(test_cases, repeat=False)) - - # They must be randomised: - assert test_cases != out - - # But all of them must have been run - assert test_cases == sorted(out) - - -def test_that_shuffle_equalises_time_across_tests(): - from mock import patch - faketime = [0.0] - - def mytime(): - return faketime[0] - - test_cases = [ - ("test1", 20), - ("test2", 10), - ("test3", 5), - ] - - time_spent_in_test = { - "test1": 0, - "test2": 0, - "test3": 0, - } - - def fake_run_test(testcase): - time_spent_in_test[testcase[0]] += testcase[1] - faketime[0] += testcase[1] - - with patch('time.time', mytime): - generator = shuffle(test_cases) - while faketime[0] < 100000: - fake_run_test(generator.next()) - - print time_spent_in_test - - assert 30000 < time_spent_in_test["test1"] < 36000 - assert 30000 < time_spent_in_test["test2"] < 36000 - assert 30000 < time_spent_in_test["test3"] < 36000 - - -def _find_file(path, root=os.path.dirname(os.path.abspath(__file__))): - return os.path.join(root, path) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/static/edit-testrun.js stb-tester-31/stbt-batch.d/static/edit-testrun.js --- stb-tester-30-5-gbefe47c/stbt-batch.d/static/edit-testrun.js 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/static/edit-testrun.js 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -$(document).ready(function() { - // http://vitalets.github.io/x-editable/docs.html - $.fn.editable.defaults.mode = "inline"; - $("#failure-reason").editable({ - type: "textarea", - url: "failure-reason", - send: "always", - toggle: "dblclick", - emptytext: " ", - success: function(response, _) { - parent.$("tr.info > td:eq(3) > span").text(truncate(response, 30)); - return {newValue: response}; - }, - }); - $("#notes").editable({ - type: "textarea", - url: "notes", - send: "always", - toggle: "dblclick", - emptytext: " ", - success: function(response, _) { - parent.$("tr.info > td:eq(4)").text(truncate(response, 30)); - return {newValue: response}; - }, - }); - // http://getbootstrap.com/2.3.2/javascript.html#tooltips - $("#failure-reason, #notes").tooltip({ - title: "Double-click to edit", - }); - - var del = $( - "[Hide this result]"); - del.on("click", function() { - if (confirm("Are you sure?")) { - $.post( - "delete", - function() { - del.replaceWith( - "(Deleted)"); - parent.$('tr.info').remove(); - }); - } - return false; - }); - $("#permalink").after(del); -}); - -function truncate(str, len) { - return str.length > len ? str.substr(0, len) + "..." : str; -} diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/templates/directory-index.html stb-tester-31/stbt-batch.d/templates/directory-index.html --- stb-tester-30-5-gbefe47c/stbt-batch.d/templates/directory-index.html 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/templates/directory-index.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ - - - - - {{dir}} - - - - -

{{f}}

- - diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/templates/index.html stb-tester-31/stbt-batch.d/templates/index.html --- stb-tester-30-5-gbefe47c/stbt-batch.d/templates/index.html 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/templates/index.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,397 +0,0 @@ - - - - - Test results {{runs[0].timestamp}} - {{runs[-1].timestamp}} - - - - - - - - -

- - -

- -

Percentage ranges reflect the fact that the true - reliability of a test is better known after running more repetitions. - The ranges are calculated using the - - Wilson Score Interval at a confidence level of 95%. They are only - valid for independent test runs.

- - - - - - - - - - {% for column in extra_columns %} - - {% endfor %} - - - - {% for run in runs %} - - - - - - - - {% for column in extra_columns %} - - {% endfor %} - - {% endfor %} - -

Timestamp	Test	Commit	Exit status	Notes	Duration	{{column}}
{{run.timestamp}}	- {% if run.exit_status != "still running" %} - - {% endif %} - {{run.test_name}} {{run.test_args}} - {% if run.exit_status != "still running" %} - - {% endif %} -	{{run.git_commit}}	- {{run.exit_status}} - {% if run.failure_reason not in ("", "success") %} - — {{run.failure_reason\|e}} - {% endif %} -	{{run.notes\|e}}	{{run.duration}}	{{ - "\n".join(run.extra_columns.get(column, "")) - }}

- -

- - - - - - - - - - - - - - - diff -Nru stb-tester-30-5-gbefe47c/stbt-batch.d/templates/testrun.html stb-tester-31/stbt-batch.d/templates/testrun.html --- stb-tester-30-5-gbefe47c/stbt-batch.d/templates/testrun.html 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-batch.d/templates/testrun.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,83 +0,0 @@ - - - - - {{run.timestamp}} {{run.test_name}} @{{run.git_commit}} - - - - - - - -

- -

[Link to this result] -{% if run.video %} - -{% endif %} -

Timestamp: {{run.timestamp}} - -

Test: {{run.test_name}} @{{run.git_commit}} -{% if run.test_args %} -

Command-line arguments:

-{% endif %} - -

Duration: {{run.duration}} -

Exit status: {{run.exit_status}} - -{% if run.exit_status > 0 %} -

Failure reason:

-{% endif %} - -{% for key in run.extra_columns %} -

{{key}}: {{ - "\n".join(run.extra_columns[key]) - }}

-{% endfor %} - -

Notes:

- -

-
- -
{{image}}
-
-

- -

Logs: -{% for f in run.files %} -

{{f}} -{% endfor %} - -

- - - - - - - - - diff -Nru stb-tester-30-5-gbefe47c/stbt-camera stb-tester-31/stbt-camera --- stb-tester-30-5-gbefe47c/stbt-camera 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -#!/bin/sh - -# Copyright 2014 stb-tester.com Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -#/ usage: stbt camera [--help] [args] -#/ -#/ Available commands are: -#/ calibrate Configure stb-tester to use a camera pointing at a TV for -#/ input -#/ validate Measure the quality of camera input/calibration -#/ -#/ For help on a specific command do 'stbt camera --help'. -#/ See 'man stbt' for more detailed information. - -this_dir=$(dirname "$0") -export PYTHONPATH="$this_dir:$PYTHONPATH" - -usage() { grep '^#/' "$0" | cut -c4-; } - -[ $# -ge 1 ] || { usage >&2; exit 1; } -cmd=$1 -shift -case "$cmd" in - -h|--help) - usage; exit 0;; - calibrate) - exec "$this_dir"/stbt-camera.d/stbt_camera_calibrate.py "$@";; - validate) - exec "$this_dir"/stbt-run "$this_dir"/stbt-camera.d/stbt_camera_validate.py "$@";; - *) - usage >&2; exit 1;; -esac diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/colours.svg stb-tester-31/stbt-camera.d/colours.svg --- stb-tester-30-5-gbefe47c/stbt-camera.d/colours.svg 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/colours.svg 1970-01-01 00:00:00.000000000 +0000 @@ -1,163 +0,0 @@ - - - - diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/glyphs.svg.jinja2 stb-tester-31/stbt-camera.d/glyphs.svg.jinja2 --- stb-tester-30-5-gbefe47c/stbt-camera.d/glyphs.svg.jinja2 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/glyphs.svg.jinja2 1970-01-01 00:00:00.000000000 +0000 @@ -1,51 +0,0 @@ - - diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/.gitignore stb-tester-31/stbt-camera.d/gst/.gitignore --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/.gitignore 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -*.so diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/plugin.c stb-tester-31/stbt-camera.d/gst/plugin.c --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/plugin.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/plugin.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,46 +0,0 @@ -/* stb-tester - * Copyright (C) 2014 stb-tester.com Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, - * Boston, MA 02110-1335, USA. - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#define PACKAGE "stb-tester" -#define PACKAGE_NAME "stb-tester" -#define GST_PACKAGE_ORIGIN "http://stb-tester.com/" - -#include - -#include "stbtgeometriccorrection.h" -#include "stbtcontraststretch.h" - -static gboolean -plugin_init (GstPlugin * plugin) -{ - return gst_element_register (plugin, "stbtgeometriccorrection", GST_RANK_NONE, - STBT_TYPE_GEOMETRIC_CORRECTION) - && gst_element_register (plugin, "stbtcontraststretch", GST_RANK_NONE, - STBT_TYPE_CONTRAST_STRETCH); -} - -GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - stbt, - "stb-tester GStreamer plugins", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.c stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.c --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,399 +0,0 @@ -/* stb-tester - * Copyright (C) 2014 stb-tester.com Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, - * Boston, MA 02110-1335, USA. - */ -/** - * SECTION:element-stbtcontraststretch - * - * Applies a vignette correction to the input image. - * - * - * Example launch line - * |[ - * gst-launch -v v4l2src ! videoconvert \ - * ! contraststretch reference-image="reference.png" - * ! videoconvert ! autoimagesink - * ]| - * - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include - -#include - -#include "stbtcontraststretch.h" -#include "stbtcontraststretch_orc.h" - -GST_DEBUG_CATEGORY_STATIC (stbt_contrast_stretch_debug_category); -#define GST_CAT_DEFAULT stbt_contrast_stretch_debug_category - -#define STBT_CONTRAST_STRETCH_LATENCY (40*GST_MSECOND) - -/* prototypes */ - -static void stbt_contrast_stretch_finalize (GObject * object); - -static void stbt_contrast_stretch_set_property (GObject * object, - guint property_id, const GValue * value, GParamSpec * pspec); -static void stbt_contrast_stretch_get_property (GObject * object, - guint property_id, GValue * value, GParamSpec * pspec); -static gboolean stbt_contrast_stretch_query (GstBaseTransform * trans, - GstPadDirection direction, GstQuery * query); - -static GstFlowReturn stbt_contrast_stretch_transform_frame ( - GstVideoFilter * filter, GstVideoFrame * frame, GstVideoFrame * out_frame); - -enum -{ - PROP_0, - PROP_BLACK_REFERENCE_IMAGE, - PROP_WHITE_REFERENCE_IMAGE, -}; - -/* pad templates */ - -#define VIDEO_SRC_CAPS \ - "video/x-raw, format=(string)BGR" - -#define VIDEO_SINK_CAPS \ - "video/x-raw, format=(string)BGR" - -/* Property defaults - No correction */ - -#define DEFAULT_REFERENCE_IMAGE "" - -/* class initialization */ - -G_DEFINE_TYPE_WITH_CODE (StbtContrastStretch, stbt_contrast_stretch, - GST_TYPE_VIDEO_FILTER, - GST_DEBUG_CATEGORY_INIT (stbt_contrast_stretch_debug_category, - "stbtcontraststretch", 0, "debug category for contraststretch element")); - -static void -stbt_contrast_stretch_class_init (StbtContrastStretchClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstBaseTransformClass *base_transform_class = - GST_BASE_TRANSFORM_CLASS (klass); - GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass); - - /* Setting up pads and setting metadata should be moved to - base_class_init if you intend to subclass this class. */ - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SRC_CAPS))); - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SINK_CAPS))); - - gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "Contrast Stretch", "Generic", - "Fixes differences in brightness across an image", - "William Manley "); - - gobject_class->finalize = stbt_contrast_stretch_finalize; - gobject_class->set_property = stbt_contrast_stretch_set_property; - gobject_class->get_property = stbt_contrast_stretch_get_property; - base_transform_class->query = GST_DEBUG_FUNCPTR (stbt_contrast_stretch_query); - video_filter_class->transform_frame = - GST_DEBUG_FUNCPTR (stbt_contrast_stretch_transform_frame); - - g_object_class_install_property (gobject_class, PROP_BLACK_REFERENCE_IMAGE, - g_param_spec_string ("black-reference-image", "Black Reference Image", - "Image taken of plain black to use as a reference", - DEFAULT_REFERENCE_IMAGE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT - | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_WHITE_REFERENCE_IMAGE, - g_param_spec_string ("white-reference-image", "White Reference Image", - "Image taken of plain white to use as a reference", - DEFAULT_REFERENCE_IMAGE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT - | G_PARAM_STATIC_STRINGS)); -} - -static void -stbt_contrast_stretch_init (StbtContrastStretch * contraststretch) -{ - g_mutex_init(&contraststretch->mutex); -} - -static void -stbt_contrast_stretch_finalize (GObject * object) -{ - StbtContrastStretch * contraststretch = STBT_CONTRAST_STRETCH (object); - g_mutex_clear(&contraststretch->mutex); - G_OBJECT_CLASS (stbt_contrast_stretch_parent_class)->finalize (object); -} - -static GstSample* -stbt_contrast_stretch_load_png (const gchar *filename, const GstCaps *caps) -{ - GstElement *src = NULL, *sink = NULL; - GstSample *out = NULL; - GstElement *pipeline = NULL; - - pipeline = gst_parse_launch ( - "filesrc name=src ! pngdec ! videoconvert ! appsink name=sink", NULL); - if (!pipeline) - goto exit; - - src = gst_bin_get_by_name(GST_BIN(pipeline), "src"); - sink = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); - if ((src == NULL) || (sink == NULL)) - goto exit; - g_object_set(src, "location", filename, NULL); - g_object_set(sink, "caps", caps, NULL); - - gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING); - - out = gst_app_sink_pull_preroll(GST_APP_SINK (sink)); - -exit: - if (pipeline) - gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL); - g_clear_object(&src); - g_clear_object(&sink); - g_clear_object(&pipeline); - - return out; -} - -inline static guint8 -max3(guint8 a, guint8 b, guint8 c) -{ - if (a > b) - return a > c ? a : c; - else - return b > c ? b : c; -} - -inline static guint8 -min3(guint8 a, guint8 b, guint8 c) -{ - if (a < b) - return a < c ? a : c; - else - return b < c ? b : c; -} - -static GstStaticCaps mycaps = GST_STATIC_CAPS ("video/x-raw,format=BGR"); - -#define SWAP(a, b) { typeof(a) _swap_tmp = a; a = b; b = _swap_tmp; } - -static void -stbt_contrast_stretch_update_coefficients(StbtContrastStretch *contraststretch, - const gchar* filename_black, const gchar* filename_white) -{ - GstSample *image[2] = {NULL, NULL}; - gchar *reference_image_name[2] = {NULL, NULL}; - gboolean success[2] = {FALSE, FALSE}; - GstMapInfo map[2] = {{0}}; - size_t idx, count = 1280*720*3; - unsigned short *coefficients = NULL; - unsigned char *offsets = NULL; - - reference_image_name[IMAGE_WHITE] = g_strdup(filename_white); - reference_image_name[IMAGE_BLACK] = g_strdup(filename_black); - - if (filename_black && filename_black[0] != '\0') { - image[IMAGE_BLACK] = stbt_contrast_stretch_load_png(filename_black, - gst_static_caps_get(&mycaps)); - - if (image[IMAGE_BLACK]) { - success[IMAGE_BLACK] = gst_buffer_map(gst_sample_get_buffer( - image[IMAGE_BLACK]), &map[IMAGE_BLACK], GST_LOCK_FLAG_READ); - g_return_if_fail (success[IMAGE_BLACK]); - } - } - - if (filename_white && filename_white[0] != '\0') { - image[IMAGE_WHITE] = stbt_contrast_stretch_load_png(filename_white, - gst_static_caps_get(&mycaps)); - if (image[IMAGE_WHITE]) { - success[IMAGE_WHITE] = gst_buffer_map(gst_sample_get_buffer( - image[IMAGE_WHITE]), &map[IMAGE_WHITE], GST_LOCK_FLAG_READ); - g_return_if_fail (success[IMAGE_WHITE]); - } - } - - if (success[IMAGE_WHITE]) - count = map[IMAGE_WHITE].size; - if (success[IMAGE_BLACK]) - count = map[IMAGE_BLACK].size; - if (success[IMAGE_WHITE] && success[IMAGE_BLACK] - && map[IMAGE_WHITE].size != map[IMAGE_BLACK].size) { - GST_ELEMENT_ERROR(contraststretch, RESOURCE, FAILED, NULL, - ("Reference image sizes don't match")); - goto error; - } - - coefficients = g_malloc(count * sizeof(unsigned short)); - offsets = g_malloc(count * sizeof(unsigned char)); - for (idx = 0; idx < count; idx += 3) { - guint8 white_point = success[IMAGE_WHITE] ? max3(map[IMAGE_WHITE].data[idx + 0], - map[IMAGE_WHITE].data[idx + 1], map[IMAGE_WHITE].data[idx + 2]) : 255; - guint8 black_point = success[IMAGE_BLACK] ? min3(map[IMAGE_BLACK].data[idx + 0], - map[IMAGE_BLACK].data[idx + 1], map[IMAGE_BLACK].data[idx + 2]) : 0; - short diff = (short) white_point - (short) black_point; - - offsets[idx] = black_point; - offsets[idx+1] = black_point; - offsets[idx+2] = black_point; - if (diff <= 0) { - coefficients[idx] = 255<<8; - coefficients[idx+1] = 255<<8; - coefficients[idx+2] = 255<<8; - } else { - coefficients[idx] = ((unsigned short)255<<8)/diff; - coefficients[idx+1] = ((unsigned short)255<<8)/diff; - coefficients[idx+2] = ((unsigned short)255<<8)/diff; - } - } - if (success[IMAGE_BLACK]) { - gst_buffer_unmap (gst_sample_get_buffer(image[IMAGE_BLACK]), &map[IMAGE_BLACK]); - gst_sample_unref (image[IMAGE_BLACK]); - } - if (success[IMAGE_WHITE]) { - gst_buffer_unmap (gst_sample_get_buffer(image[IMAGE_WHITE]), &map[IMAGE_WHITE]); - gst_sample_unref (image[IMAGE_WHITE]); - } -error: - /* Want to be doing as little as possible with the mutex locked so use SWAP - */ - g_mutex_lock (&contraststretch->mutex); - SWAP(contraststretch->reference_image_name[IMAGE_WHITE], - reference_image_name[IMAGE_WHITE]); - SWAP(contraststretch->reference_image_name[IMAGE_BLACK], - reference_image_name[IMAGE_BLACK]); - SWAP(contraststretch->coefficient_count, count); - SWAP(contraststretch->coefficients, coefficients); - SWAP(contraststretch->offsets, offsets); - g_mutex_unlock (&contraststretch->mutex); - - g_free (reference_image_name[IMAGE_WHITE]); - g_free (reference_image_name[IMAGE_BLACK]); - g_free (coefficients); - g_free (offsets); -} - -static void -stbt_contrast_stretch_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec) -{ - StbtContrastStretch *contraststretch = STBT_CONTRAST_STRETCH (object); - - GST_DEBUG_OBJECT (contraststretch, "set_property"); - switch (property_id) { - case PROP_BLACK_REFERENCE_IMAGE: - stbt_contrast_stretch_update_coefficients (contraststretch, - g_strdup(g_value_get_string (value)), contraststretch->reference_image_name[IMAGE_WHITE]); - break; - case PROP_WHITE_REFERENCE_IMAGE: - stbt_contrast_stretch_update_coefficients (contraststretch, - contraststretch->reference_image_name[IMAGE_BLACK], g_strdup(g_value_get_string (value))); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } -} - -static void -stbt_contrast_stretch_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec) -{ - StbtContrastStretch *contraststretch = STBT_CONTRAST_STRETCH (object); - - GST_DEBUG_OBJECT (contraststretch, "get_property"); - - switch (property_id) { - case PROP_BLACK_REFERENCE_IMAGE: - if (contraststretch->reference_image_name[IMAGE_BLACK]) - g_value_set_string(value, ""); - else - g_value_set_string(value, contraststretch->reference_image_name[IMAGE_BLACK]); - break; - case PROP_WHITE_REFERENCE_IMAGE: - if (contraststretch->reference_image_name[IMAGE_WHITE]) - g_value_set_string(value, ""); - else - g_value_set_string(value, contraststretch->reference_image_name[IMAGE_WHITE]); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } -} - -static gboolean -stbt_contrast_stretch_query (GstBaseTransform * trans, GstPadDirection direction, - GstQuery * query) -{ - gboolean result = GST_BASE_TRANSFORM_CLASS (stbt_contrast_stretch_parent_class) - ->query (trans, direction, query); - - if (result && GST_QUERY_TYPE (query) == GST_QUERY_LATENCY) { - GstClockTime min, max; - gboolean live; - - gst_query_parse_latency (query, &live, &min, &max); - - min += STBT_CONTRAST_STRETCH_LATENCY; - max += STBT_CONTRAST_STRETCH_LATENCY; - - gst_query_set_latency (query, live, min, max); - } - - return result; -} - -/* transform */ -static GstFlowReturn -stbt_contrast_stretch_transform_frame (GstVideoFilter * filter, - GstVideoFrame * in_frame, GstVideoFrame * out_frame) -{ - StbtContrastStretch *contraststretch = STBT_CONTRAST_STRETCH (filter); - int len; - - GST_DEBUG_OBJECT (contraststretch, "transform_frame"); - - g_mutex_lock (&contraststretch->mutex); - - if (contraststretch->coefficients == NULL) - goto done; - len = in_frame->info.height * in_frame->info.width * 3; - if (contraststretch->coefficient_count != len) { - GST_ELEMENT_ERROR(contraststretch, STREAM, WRONG_TYPE, NULL, - ("Wrong size")); - goto error; - } - - /* Defined in stbtcontraststretch.orc for speed */ - stbt_contraststretch_apply(out_frame->data[0], in_frame->data[0], contraststretch->offsets, - contraststretch->coefficients, contraststretch->coefficient_count); - - g_mutex_unlock (&contraststretch->mutex); - -done: - return GST_FLOW_OK; -error: - return GST_FLOW_ERROR; -} diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.h stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.h --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -/* stb-tester - * Copyright (C) 2014 stb-tester.com Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#ifndef _STBT_CONTRAST_STRETCH_H_ -#define _STBT_CONTRAST_STRETCH_H_ - -#include -#include - -G_BEGIN_DECLS -#define STBT_TYPE_CONTRAST_STRETCH (stbt_contrast_stretch_get_type()) -#define STBT_CONTRAST_STRETCH(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),STBT_TYPE_CONTRAST_STRETCH,StbtContrastStretch)) -#define STBT_CONTRAST_STRETCH_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),STBT_TYPE_CONTRAST_STRETCH,StbtContrastStretchClass)) -#define STBT_IS_CONTRAST_STRETCH(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),STBT_TYPE_CONTRAST_STRETCH)) -#define STBT_IS_CONTRAST_STRETCH_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),STBT_TYPE_CONTRAST_STRETCH)) -typedef struct _StbtContrastStretch StbtContrastStretch; -typedef struct _StbtContrastStretchClass StbtContrastStretchClass; - -enum { - IMAGE_BLACK, - IMAGE_WHITE, -}; - -struct _StbtContrastStretch -{ - GstVideoFilter base_contraststretch; - - GMutex mutex; - - gchar *reference_image_name[2]; - size_t coefficient_count; - - /* Take this number away from the pixel value to make it black */ - unsigned char *offsets; - - /* coefficients by which each pixel value must be multiplied as a fixed point - with the decimal point at bit 8. */ - unsigned short *coefficients; -}; - -struct _StbtContrastStretchClass -{ - GstVideoFilterClass base_contraststretch_class; -}; - -GType stbt_contrast_stretch_get_type (void); - -G_END_DECLS -#endif diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.orc stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.orc --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtcontraststretch.orc 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/stbtcontraststretch.orc 1970-01-01 00:00:00.000000000 +0000 @@ -1,18 +0,0 @@ -.function stbt_contraststretch_apply -.dest 1 out -.source 1 data -.source 1 offset -# Coefficient is 16 bits fixed point with 8 bits after the decimal place -.source 2 coefficient -.temp 1 tmp_8_0 -.temp 2 tmp_16_0 -.temp 4 out_32_8 -.temp 4 out_32_0 -.temp 2 out_16_0 - -subusb tmp_8_0, data, offset # tmp_8_0 = data - offset -convubw tmp_16_0, tmp_8_0 # tmp_16_0 = (uint16) tmp_8_0 -muluwl out_32_8, tmp_16_0, coefficient # out_24_8 = tmp_16_0 * coefficient -shrul out_32_0, out_32_8, 8 # out_32_0 = out_32_8 >> 8 -convlw out_16_0, out_32_0 # out_16_0 = (uint16) out_32_0 -convsuswb out, out_16_0 # out = (uint8) clamp(out_16_0) diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtgeometriccorrection.c stb-tester-31/stbt-camera.d/gst/stbtgeometriccorrection.c --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtgeometriccorrection.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/stbtgeometriccorrection.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,437 +0,0 @@ -/* stb-tester - * Copyright (C) 2014 stb-tester.com Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, - * Boston, MA 02110-1335, USA. - */ -/** - * SECTION:element-stbtgeometriccorrection - * - * The geometriccorrection element supports videoing TVs with webcams. Based on data - * about the camera and the location of the TV in the image it will output what - * is being shown on the TV. - * - * - * Example launch line - * |[ - * gst-launch -v v4l2src ! video/x-raw,width=1080,height=720 ! videoconvert \ - * ! geometriccorrection camera-matrix="1650.9498393436943 0.0 929.0144682177463 0.0 1650.0877603121983 558.55148343511712 0.0 0.0 1.0" \ - * distortion-coefficients="0.10190074279957002 -0.18199960394082984 0.00054709946447197304 0.00039158751563262209 -0.0479346424607551" \ - * inv-homography-matrix="1363.1288390747131 7.438063312178901 643.25417659039442 38.730496461642169 1354.6127277059686 388.87669488396085 0.040142377232467538 0.043862451595691403 1.0" \ - * ! videoconvert ! autoimagesink - * ]| - * Captures a video of a TV from your webcam and shows you a video of what's - * on TV. - * - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include - -#include -#include -#include - -#include "stbtgeometriccorrection.h" - -GST_DEBUG_CATEGORY_STATIC (stbt_geometric_correction_debug_category); -#define GST_CAT_DEFAULT stbt_geometric_correction_debug_category - -#define STBT_GEOMETRIC_CORRECTION_LATENCY (40*GST_MSECOND) - -/* prototypes */ -static void stbt_geometric_correction_finalize (GObject * gobject); - -static void stbt_geometric_correction_set_property (GObject * object, - guint property_id, const GValue * value, GParamSpec * pspec); -static void stbt_geometric_correction_get_property (GObject * object, - guint property_id, GValue * value, GParamSpec * pspec); -static gboolean stbt_geometric_correction_query (GstBaseTransform * trans, - GstPadDirection direction, GstQuery * query); - -static gboolean stbt_geometric_correction_start (GstBaseTransform * trans); -static gboolean stbt_geometric_correction_stop (GstBaseTransform * trans); -static GstCaps *stbt_watch_transform_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, GstCaps * filter); - -static GstFlowReturn stbt_geometric_correction_transform_frame (GstVideoFilter * filter, - GstVideoFrame * inframe, GstVideoFrame * outframe); - -enum -{ - PROP_0, - PROP_CAMERA_MATRIX, - PROP_DISTORTION_COEFFICIENTS, - PROP_INV_HOMOGRAPHY_MATRIX -}; - -/* pad templates */ - -#define VIDEO_SRC_CAPS \ - "video/x-raw, format=(string)BGR, width=(int)1280, height=(int)720" - -#define VIDEO_SINK_CAPS \ - "video/x-raw, format=(string)BGR, width=(int)1920, height=(int)1080" - -/* Property defaults - Scales 1920x1080 to 1280x720, the equivalent of a noop - for this element */ - -#define DEFAULT_CAMERA_MATRIX \ - "1.0 0.0 0.0 " \ - "0.0 1.0 0.0 " \ - "0.0 0.0 1.0 " - -#define DEFAULT_DISTORTION_COEFFICIENTS \ - "0.0 0.0 0.0 0.0 0.0" - -#define DEFAULT_INV_HOMOGRAPHY_MATRIX \ - "1.5 0.0 0.25 " \ - "0.0 1.5 0.25 " \ - "0.0 0.0 1.0 " - -/* class initialization */ - -G_DEFINE_TYPE_WITH_CODE (StbtGeometricCorrection, stbt_geometric_correction, - GST_TYPE_VIDEO_FILTER, - GST_DEBUG_CATEGORY_INIT (stbt_geometric_correction_debug_category, "stbtgeometriccorrection", - 0, "debug category for geometriccorrection element")); - -static void -stbt_geometric_correction_class_init (StbtGeometricCorrectionClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstBaseTransformClass *base_transform_class = - GST_BASE_TRANSFORM_CLASS (klass); - GstVideoFilterClass *video_filter_class = GST_VIDEO_FILTER_CLASS (klass); - - /* Setting up pads and setting metadata should be moved to - base_class_init if you intend to subclass this class. */ - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SRC_CAPS))); - gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), - gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - gst_caps_from_string (VIDEO_SINK_CAPS))); - - gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "Geometric Correction", "Generic", - "Input: pictures of a TV output: what is playing on that TV", - "William Manley "); - - gobject_class->set_property = GST_DEBUG_FUNCPTR (stbt_geometric_correction_set_property); - gobject_class->get_property = GST_DEBUG_FUNCPTR (stbt_geometric_correction_get_property); - gobject_class->finalize = GST_DEBUG_FUNCPTR (stbt_geometric_correction_finalize); - - base_transform_class->start = GST_DEBUG_FUNCPTR (stbt_geometric_correction_start); - base_transform_class->stop = GST_DEBUG_FUNCPTR (stbt_geometric_correction_stop); - base_transform_class->query = GST_DEBUG_FUNCPTR (stbt_geometric_correction_query); - base_transform_class->transform_caps = - GST_DEBUG_FUNCPTR (stbt_watch_transform_caps); - video_filter_class->transform_frame = - GST_DEBUG_FUNCPTR (stbt_geometric_correction_transform_frame); - - g_object_class_install_property (gobject_class, PROP_CAMERA_MATRIX, - g_param_spec_string ("camera-matrix", "Camera Matrix", - "The camera matrix of the camera used to capture the input", - DEFAULT_CAMERA_MATRIX, G_PARAM_READWRITE | G_PARAM_CONSTRUCT)); - g_object_class_install_property (gobject_class, PROP_DISTORTION_COEFFICIENTS, - g_param_spec_string ("distortion-coefficients", "Distortion Coefficients", - "The distortion coefficients of the camera used to capture the input", - DEFAULT_DISTORTION_COEFFICIENTS, - G_PARAM_READWRITE | G_PARAM_CONSTRUCT)); - g_object_class_install_property (gobject_class, PROP_INV_HOMOGRAPHY_MATRIX, - g_param_spec_string ("inv-homography-matrix", "Homography Matrix", - "The inverse homography matrix describing the region of interest", - DEFAULT_INV_HOMOGRAPHY_MATRIX, G_PARAM_READWRITE | G_PARAM_CONSTRUCT)); -} - -static void -stbt_geometric_correction_init (StbtGeometricCorrection * geometriccorrection) -{ - g_mutex_init(&geometriccorrection->props_mutex); -} - -static void -stbt_geometric_correction_finalize (GObject * object) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (object); - g_mutex_clear(&geometriccorrection->props_mutex); - G_OBJECT_CLASS (stbt_geometric_correction_parent_class)->finalize (object); -} - -static void -stbt_geometric_correction_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (object); - int values_read; - float (*m)[3] = NULL; - float *d; - - GST_DEBUG_OBJECT (geometriccorrection, "set_property"); - - g_mutex_lock(&geometriccorrection->props_mutex); - geometriccorrection->needs_regen = TRUE; - switch (property_id) { - case PROP_CAMERA_MATRIX: - m = geometriccorrection->camera_matrix; - case PROP_INV_HOMOGRAPHY_MATRIX: - m = m ? m : geometriccorrection->inv_homography_matrix; - - values_read = sscanf (g_value_get_string (value), - "%f %f %f %f %f %f %f %f %f", - &m[0][0], &m[0][1], &m[0][2], - &m[1][0], &m[1][1], &m[1][2], &m[2][0], &m[2][1], &m[2][2]); - g_warn_if_fail (values_read == 9); - break; - case PROP_DISTORTION_COEFFICIENTS: - d = geometriccorrection->distortion_coefficients; - values_read = sscanf (g_value_get_string (value), - "%f %f %f %f %f", &d[0], &d[1], &d[2], &d[3], &d[4]); - g_warn_if_fail (values_read == 5); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } - geometriccorrection->needs_regen = TRUE; - g_mutex_unlock(&geometriccorrection->props_mutex); -} - -static void -stbt_geometric_correction_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (object); - float (*m)[3] = NULL; - float *d; - gchar *str = NULL; - - GST_DEBUG_OBJECT (geometriccorrection, "get_property"); - - g_mutex_lock(&geometriccorrection->props_mutex); - switch (property_id) { - case PROP_CAMERA_MATRIX: - m = geometriccorrection->camera_matrix; - case PROP_INV_HOMOGRAPHY_MATRIX: - m = m ? m : geometriccorrection->inv_homography_matrix; - - str = g_strdup_printf ("%f %f %f %f %f %f %f %f %f", - m[0][0], m[0][1], m[0][2], - m[1][0], m[1][1], m[1][2], m[2][0], m[2][1], m[2][2]); - g_value_take_string (value, str); - break; - case PROP_DISTORTION_COEFFICIENTS: - d = geometriccorrection->distortion_coefficients; - str = g_strdup_printf ("%f %f %f %f %f", d[0], d[1], d[2], d[3], d[4]); - g_value_take_string (value, str); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); - return; - } - g_mutex_unlock(&geometriccorrection->props_mutex); -} - -static gboolean -stbt_geometric_correction_query (GstBaseTransform * trans, GstPadDirection direction, - GstQuery * query) -{ - gboolean result = GST_BASE_TRANSFORM_CLASS (stbt_geometric_correction_parent_class) - ->query (trans, direction, query); - - if (result && GST_QUERY_TYPE (query) == GST_QUERY_LATENCY) { - GstClockTime min, max; - gboolean live; - - gst_query_parse_latency (query, &live, &min, &max); - - min += STBT_GEOMETRIC_CORRECTION_LATENCY; - max += STBT_GEOMETRIC_CORRECTION_LATENCY; - - gst_query_set_latency (query, live, min, max); - } - - return result; -} - -typedef struct Coord_ -{ - float x; - float y; -} Coord; - -typedef struct Coord3_ -{ - float x; - float y; - float z; -} Coord3; - -static void -regenerate_remapping_matrix (StbtGeometricCorrection * geometriccorrection) -{ - int x, y; - CvMat *remapping = cvCreateMat (720, 1280, CV_32FC2); - CvMat *remapping_int = cvCreateMat (720, 1280, CV_16SC2); - CvMat *remapping_interpolation = cvCreateMat (720, 1280, CV_16UC1); - CvMat remapping_flat = cvMat (1, 1280 * 720, CV_32FC2, remapping->data.fl); - CvMat *temp; - CvMat camera_matrix = cvMat (3, 3, CV_32F, geometriccorrection->camera_matrix); - CvMat distortion_coefficients = - cvMat (5, 1, CV_32F, geometriccorrection->distortion_coefficients); - CvMat inv_homography_matrix = cvMat (3, 3, CV_32F, geometriccorrection->inv_homography_matrix); - float no_transform_data[3] = { 0.0, 0.0, 0.0 }; - CvMat no_transform = cvMat (3, 1, CV_32F, no_transform_data); - - for (x = 0; x < 1280; x++) { - for (y = 0; y < 720; y++) { - CV_MAT_ELEM (*remapping, Coord, y, x).x = x; - CV_MAT_ELEM (*remapping, Coord, y, x).y = y; - } - } - - /* remap takes a matrix of (x, y) coordinates. The value of any pixel in the - output will be taken from the input image at the coordinates in the - remapping matrix. Therefore we need to modify the coordinates in the - "identity" mapping created above until they point at the locations in the - source image that we want to get the values from. - - By transforming coordinates from dest to src remap will transform values - from src to dest. */ - cvPerspectiveTransform ((const CvArr *) remapping, (CvArr *) remapping, - &inv_homography_matrix); - - temp = cvCreateMat (1, 1280 * 720, CV_32FC3); - for (x = 0; x < 1280; x++) { - for (y = 0; y < 720; y++) { - CV_MAT_ELEM (*temp, Coord3, 0, x + 1280 * y).x = - CV_MAT_ELEM (*remapping, Coord, y, x).x; - CV_MAT_ELEM (*temp, Coord3, 0, x + 1280 * y).y = - CV_MAT_ELEM (*remapping, Coord, y, x).y; - CV_MAT_ELEM (*temp, Coord3, 0, x + 1280 * y).z = 0.0; - } - } - - cvProjectPoints2 ((const CvArr *) temp, &no_transform, &no_transform, - &camera_matrix, &distortion_coefficients, (CvArr *) & remapping_flat, - NULL, NULL, NULL, NULL, NULL, 0); - cvReleaseMat (&temp); - - cvConvertMaps (remapping, NULL, remapping_int, remapping_interpolation); - cvReleaseMat (&remapping); - - cvReleaseMat (&geometriccorrection->remapping_int); - cvReleaseMat (&geometriccorrection->remapping_interpolation); - geometriccorrection->remapping_int = remapping_int; - geometriccorrection->remapping_interpolation = remapping_interpolation; -} - -static gboolean -stbt_geometric_correction_start (GstBaseTransform * trans) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (trans); - - GST_DEBUG_OBJECT (geometriccorrection, "start"); - - g_return_val_if_fail (geometriccorrection->remapping_int == NULL, FALSE); - g_return_val_if_fail (geometriccorrection->remapping_interpolation == NULL, FALSE); - regenerate_remapping_matrix (geometriccorrection); - - return TRUE; -} - -static gboolean -stbt_geometric_correction_stop (GstBaseTransform * trans) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (trans); - - GST_DEBUG_OBJECT (geometriccorrection, "stop"); - - cvReleaseMat (&geometriccorrection->remapping_int); - cvReleaseMat (&geometriccorrection->remapping_interpolation); - - return TRUE; -} - -static GstCaps * -stbt_watch_transform_caps (GstBaseTransform * trans, - GstPadDirection direction, GstCaps * caps, GstCaps * filter) -{ - GstCaps *ret = gst_caps_copy (caps); - GValue res = G_VALUE_INIT; - - g_value_init (&res, G_TYPE_INT); - - if (direction == GST_PAD_SRC) { - g_value_set_int (&res, 1920); - gst_caps_set_value (ret, "width", &res); - g_value_set_int (&res, 1080); - gst_caps_set_value (ret, "height", &res); - } else { - g_value_set_int (&res, 1280); - gst_caps_set_value (ret, "width", &res); - g_value_set_int (&res, 720); - gst_caps_set_value (ret, "height", &res); - } - - if (filter) { - GstCaps *intersection; - - intersection = - gst_caps_intersect_full (filter, ret, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (ret); - ret = intersection; - } - - return ret; -} - -/* transform */ -static GstFlowReturn -stbt_geometric_correction_transform_frame (GstVideoFilter * filter, - GstVideoFrame * inframe, GstVideoFrame * outframe) -{ - StbtGeometricCorrection *geometriccorrection = STBT_GEOMETRIC_CORRECTION (filter); - CvMat inmat = cvMat (1080, 1920, CV_8UC3, inframe->data[0]); - CvMat outmat = cvMat (720, 1280, CV_8UC3, outframe->data[0]); - - GST_DEBUG_OBJECT (geometriccorrection, "transform_frame"); - - g_mutex_lock(&geometriccorrection->props_mutex); - if (geometriccorrection->needs_regen) { - regenerate_remapping_matrix(geometriccorrection); - geometriccorrection->needs_regen = FALSE; - } - g_mutex_unlock(&geometriccorrection->props_mutex); - - g_return_val_if_fail (geometriccorrection->remapping_int, GST_FLOW_ERROR); - g_return_val_if_fail (geometriccorrection->remapping_interpolation, GST_FLOW_ERROR); - g_return_val_if_fail (inframe->info.width == 1920 && - inframe->info.height == 1080, GST_FLOW_ERROR); - g_return_val_if_fail (outframe->info.width == 1280 && - outframe->info.height == 720, GST_FLOW_ERROR); - - cvRemap (&inmat, &outmat, geometriccorrection->remapping_int, - geometriccorrection->remapping_interpolation, - CV_INTER_LINEAR + CV_WARP_FILL_OUTLIERS, cvScalarAll (0)); - - return GST_FLOW_OK; -} diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtgeometriccorrection.h stb-tester-31/stbt-camera.d/gst/stbtgeometriccorrection.h --- stb-tester-30-5-gbefe47c/stbt-camera.d/gst/stbtgeometriccorrection.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/gst/stbtgeometriccorrection.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,66 +0,0 @@ -/* stb-tester - * Copyright (C) 2014 stb-tester.com Ltd. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#ifndef _STBT_GEOMETRIC_CORRECTION_H_ -#define _STBT_GEOMETRIC_CORRECTION_H_ - -#include -#include - -G_BEGIN_DECLS -#define STBT_TYPE_GEOMETRIC_CORRECTION (stbt_geometric_correction_get_type()) -#define STBT_GEOMETRIC_CORRECTION(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),STBT_TYPE_GEOMETRIC_CORRECTION,StbtGeometricCorrection)) -#define STBT_GEOMETRIC_CORRECTION_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),STBT_TYPE_GEOMETRIC_CORRECTION,StbtGeometricCorrectionClass)) -#define STBT_IS_GEOMETRIC_CORRECTION(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),STBT_TYPE_GEOMETRIC_CORRECTION)) -#define STBT_IS_GEOMETRIC_CORRECTION_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass),STBT_TYPE_GEOMETRIC_CORRECTION)) -typedef struct _StbtGeometricCorrection StbtGeometricCorrection; -typedef struct _StbtGeometricCorrectionClass StbtGeometricCorrectionClass; -typedef struct CvMat CvMat; - -struct _StbtGeometricCorrection -{ - GstVideoFilter base_geometriccorrection; - - GMutex props_mutex; - - /* Properties that describe the transformation. See the OpenCV documentation - * for more information. These are used to create remapping below: */ - float camera_matrix[3][3]; - float distortion_coefficients[5]; - float inv_homography_matrix[3][3]; - - gboolean needs_regen; - - /* A 1280x720 array of (int16, int16) and a 1280x720 array of uint16 for use - * by cvRemap. Generated based on above properties. We need two matricies - * so cvRemap can use the faster fixed-point maths rather than floating - * point. See convertMaps for more information. */ - CvMat *remapping_int; - CvMat *remapping_interpolation; -}; - -struct _StbtGeometricCorrectionClass -{ - GstVideoFilterClass base_geometriccorrection_class; -}; - -GType stbt_geometric_correction_get_type (void); - -G_END_DECLS -#endif diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/README.md stb-tester-31/stbt-camera.d/README.md --- stb-tester-30-5-gbefe47c/stbt-camera.d/README.md 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -* Point Logitech C920 webcam at TV screen. Note that the camera's firmware has - a bug where it forgets the settings set previously via V4L controls. We - [fixed](https://git.kernel.org/cgit/linux/kernel/git/next/linux-next.git/log/?qt=grep&q=uvcvideo%3A+Work+around+buggy+Logitech+C920+firmware) - this in Linux kernel 3.18 and newer. -* Run `stbt --with-experimental camera calibrate` and follow the instructions. - At the end of the calibration process this will write the calibration - settings to your stbt configuration file. -* Run `stbt --with-experimental camera validate` to test the calibration - settings. -* Use `stbt run` as usual to run your tests. - -On Fedora you'll need to build the GStreamer `voaacenc` plugin manually. It -isn't available in the Fedora or RPMFusion repositories (see -). On Ubuntu `voaacenc` is -available from the "universe" repository. We use `voaacenc` to generate the -calibration videos that are to be played on the TV. diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/stbt_camera_calibrate.py stb-tester-31/stbt-camera.d/stbt_camera_calibrate.py --- stb-tester-30-5-gbefe47c/stbt-camera.d/stbt_camera_calibrate.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/stbt_camera_calibrate.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,635 +0,0 @@ -#!/usr/bin/python -u -# Encoding: utf-8 -# pylint:disable=protected-access - -import math -import re -import readline -import subprocess -import sys -import time -from contextlib import contextmanager -from os.path import dirname - -import cv2 -import gi -import numpy - -import _stbt.camera.chessboard as chessboard -import _stbt.core -import stbt -from _stbt import tv_driver -from _stbt.config import set_config, xdg_config_dir - -gi.require_version("Gst", "1.0") -from gi.repository import Gst # pylint:disable=wrong-import-order - -COLOUR_SAMPLES = 50 -videos = {} - -# -# Geometric calibration -# - -videos['chessboard'] = chessboard.VIDEO - -arrows = list(u'←↙↓↘→↗↑↖') - - -def off_to_arrow(off): - u""" - >>> print off_to_arrow((1, 1)) - ↘ - >>> print off_to_arrow((-1, 0)) - ← - """ - if numpy.linalg.norm(off) > 0.5: - angle = math.atan2(off[1], -off[0]) - return arrows[int(angle / 2 / math.pi * len(arrows) + len(arrows) + 0.5) - % len(arrows)] - else: - return u'O' - - -# ANSI colour codes for printing progress. -OKGREEN = '\033[92m' -WARNING = '\033[93m' -FAIL = '\033[91m' -ENDC = '\033[0m' -BOLD = '\033[1m' - - -def rate(r): - """How good is the match on a scale of 0-2?""" - if r < 0.5: - return 2 - elif r < 5: - return 1 - else: - return 0 - - -def print_error_map(outstream, ideal_points, measured_points): - oldx = 0.0 - outstream.write( - BOLD + "Geometric Calibration Report:\n" + ENDC + - "\n" - " Legend:\n" - " " + OKGREEN + "O" + ENDC + " - Pixel perfect\n" - " " + WARNING + "↗" + ENDC + " - Off by up-to 5 pixels\n" - " " + FAIL + "↗" + ENDC + " - Off by more than 5 pixels\n" - "\n") - for ideal, measured in sorted(zip(ideal_points, measured_points), - key=lambda a: (-a[0][1], a[0][0])): - if ideal[0] < oldx: - outstream.write('\n') - off = ideal - measured - outstream.write( - (u"%s%s" % ([FAIL, WARNING, OKGREEN][rate(numpy.linalg.norm(off))], - off_to_arrow(off))) - .encode('utf-8')) - oldx = ideal[0] - outstream.write("\n" + ENDC) - - -def geometric_calibration(dut, tv, device, interactive=True): - tv.show('chessboard') - - sys.stdout.write("Performing Geometric Calibration\n") - - chessboard_calibration(dut) - if interactive: - while prompt_for_adjustment(device): - try: - chessboard_calibration(dut) - except chessboard.NoChessboardError: - tv.show('chessboard') - chessboard_calibration(dut) - - -def chessboard_calibration(dut, timeout=10): - from _stbt.gst_utils import array_from_sample - - undistorted_appsink = \ - dut._display.source_pipeline.get_by_name('undistorted_appsink') - - sys.stderr.write("Searching for chessboard\n") - endtime = time.time() + timeout - while time.time() < endtime: - sample = undistorted_appsink.emit('pull-sample') - try: - input_image = array_from_sample(sample) - params = chessboard.calculate_calibration_params(input_image) - break - except chessboard.NoChessboardError: - if time.time() > endtime: - raise - - geometriccorrection = dut._display.source_pipeline.get_by_name( - 'geometric_correction') - - geometriccorrection_params = { - 'camera-matrix': ('{fx} 0 {cx}' - ' 0 {fy} {cy}' - ' 0 0 1').format(**params), - 'distortion-coefficients': '{k1} {k2} {p1} {p2} {k3}'.format(**params), - 'inv-homography-matrix': ( - '{ihm11} {ihm21} {ihm31} ' - '{ihm12} {ihm22} {ihm32} ' - '{ihm13} {ihm23} {ihm33}').format(**params), - } - for key, value in geometriccorrection_params.items(): - geometriccorrection.set_property(key, value) - - print_error_map( - sys.stderr, - *chessboard.find_corrected_corners(params, input_image)) - - set_config( - 'global', 'geometriccorrection_params', - ' '.join('%s="%s"' % v for v in geometriccorrection_params.items())) - -# -# Colour Measurement -# - - -def qrc(data): - import cStringIO - import qrcode - import qrcode.image.svg - - out = cStringIO.StringIO() - qrcode.make(data, image_factory=qrcode.image.svg.SvgPathImage).save(out) - qrsvg = out.getvalue() - - return re.search('d="(.*?)"', qrsvg).group(1) - - -def generate_colours_video(): - import random - template_svg = open(dirname(__file__) + '/colours.svg', 'r').read() - for _ in range(0, 10 * 60 * 8): - colour = '#%06x' % random.randint(0, 256 ** 3) - svg = template_svg.replace('#c0ffee', colour) - svg = svg.replace("m 0,0 26,0 0,26 -26,0 z", qrc(colour)) - yield (svg, 1.0 / 8 * Gst.SECOND) - -videos['colours2'] = ('image/svg', generate_colours_video) - - -class QRScanner(object): - def __init__(self): - import zbar - self.scanner = zbar.ImageScanner() - self.scanner.parse_config('enable') - - def read_qr_codes(self, image): - import zbar - zimg = zbar.Image(image.shape[1], image.shape[0], 'Y800', - cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).tostring()) - self.scanner.scan(zimg) - return [s.data for s in zimg] - - -def analyse_colours_video(dut, number=None): - """RGB!""" - errors_in_a_row = 0 - n = 0 - qrscanner = QRScanner() - for frame in dut.frames(): - if number is not None and n >= number: - return - n = n + 1 - - # The colour is written above and below the rectangle because we want - # to be sure that the top of the colour box is from the same frame as - # the bottom. - codes = qrscanner.read_qr_codes(frame) - - if (len(codes) == 4 and re.match('#[0-9a-f]{6}', codes[0]) and - all(c == codes[0] for c in codes)): - colour_hex = codes[0] - desired = numpy.array(( - int(colour_hex[1:3], 16), - int(colour_hex[3:5], 16), - int(colour_hex[5:7], 16))) - colour = cv2.mean(frame[240:480, 520:760]) - colour = (colour[2], colour[1], colour[0]) - yield (n, desired, colour) - errors_in_a_row = 0 - else: - errors_in_a_row += 1 - if errors_in_a_row > 50: - raise RuntimeError( - "Failed to find hexidecimal colour description") - - -def avg_colour(colours): - n = len(colours) - return ( - sum([c[0] for c in colours]) / n, - sum([c[1] for c in colours]) / n, - sum([c[2] for c in colours]) / n) - - -example_v4l2_ctl_output = """\ - brightness (int) : min=-64 max=64 step=1 default=15 value=15 - contrast (int) : min=0 max=95 step=1 default=30 value=30 -""" - - -def v4l2_ctls(device, data=None): - """ - >>> import pprint - >>> pprint.pprint(dict(v4l2_ctls(None, example_v4l2_ctl_output))) - {'brightness': {'default': '15', - 'max': '64', - 'min': '-64', - 'step': '1', - 'value': '15'}, - 'contrast': {'default': '30', - 'max': '95', - 'min': '0', - 'step': '1', - 'value': '30'}} - """ - if data is None: - data = subprocess.check_output(['v4l2-ctl', '-d', device, '-l']) - for line in data.split('\n'): - vals = line.strip().split() - if vals == []: - continue - yield (vals[0], dict([v.split('=', 2) for v in vals[3:]])) - - -def setup_tab_completion(completer): - next_ = [None] - generator = [None] - - def readline_completer(text, state): - if state == 0: - generator[0] = iter(completer(text)) - next_[0] = 0 - - assert state == next_[0] - next_[0] += 1 - - try: - return generator[0].next() - except StopIteration: - return None - - readline.parse_and_bind("tab: complete") - readline.set_completer_delims("") - readline.set_completer(readline_completer) - - -def prompt_for_adjustment(device): - # Allow adjustment - subprocess.check_call(['v4l2-ctl', '-d', device, '-L']) - ctls = dict(v4l2_ctls(device)) - - def v4l_completer(text): - if text == '': - return ['yes', 'no', 'set'] - if text.startswith('set '): - return ['set ' + x + ' ' - for x in ctls.keys() if x.startswith(text[4:])] - if "set ".startswith(text.lower()): - return ["set "] - if 'yes'.startswith(text.lower()): - return ["yes"] - if 'no'.startswith(text.lower()): - return ["no"] - - setup_tab_completion(v4l_completer) - - cmd = raw_input("Happy? [Y/n/set] ").strip().lower() - if cmd.startswith('set'): - x = cmd.split(None, 2) - if len(x) != 3: - print "Didn't understand command %r" % x - else: - _, var, val = x - subprocess.check_call( - ['v4l2-ctl', '-d', device, "-c", "%s=%s" % (var, val)]) - - set_config('global', 'v4l2_ctls', ','.join( - ["%s=%s" % (c, a['value']) - for c, a in dict(v4l2_ctls(device)).items()])) - - if cmd.startswith('y') or cmd == '': - return False # We're done - else: - return True # Continue looping - - -def pop_with_progress(iterator, total, width=20, stream=sys.stderr): - stream.write('\n') - for n, v in enumerate(iterator): - if n == total: - break - progress = (n * width) // total - stream.write( - '[%s] %8d / %d\r' % ( - '#' * progress + ' ' * (width - progress), n, total)) - yield v - stream.write('\r' + ' ' * (total + 28) + '\r') - - -def fit_fn(ideals, measureds): - """ - >>> f = fit_fn([120, 240, 150, 18, 200], - ... [120, 240, 150, 18, 200]) - >>> print "%.2f %.2f" % (f(0), f(56)) - 0.00 56.00 - """ - from scipy.optimize import curve_fit - from scipy.interpolate import interp1d - POINTS = 5 - xs = [n * 255.0 / (POINTS + 1) for n in range(0, POINTS + 2)] - - def fn(x, ys): - return interp1d(xs, numpy.array([0] + ys + [255]))(x) - - ys, _ = curve_fit( # pylint:disable=unbalanced-tuple-unpacking - lambda x, *args: fn(x, list(args)), ideals, measureds, [0.0] * POINTS) - return interp1d(xs, numpy.array([0] + ys.tolist() + [255])) - - -@contextmanager -def colour_graph(dut): - if not _can_show_graphs(): - sys.stderr.write("Install matplotlib and scipy for graphical " - "assistance with colour calibration\n") - yield lambda: None - return - - from matplotlib import pyplot - sys.stderr.write('Analysing colours...\n') - pyplot.ion() - - ideals = [[], [], []] - measureds = [[], [], []] - - pyplot.figure() - - def update(): - pyplot.cla() - pyplot.axis([0, 255, 0, 255]) - pyplot.ylabel("Measured colour") - pyplot.xlabel("Ideal colour") - pyplot.grid() - - for n, ideal, measured in pop_with_progress( - analyse_colours_video(dut), COLOUR_SAMPLES): - pyplot.draw() - for c in [0, 1, 2]: - ideals[c].append(ideal[c]) - measureds[c].append(measured[c]) - pyplot.plot([ideal[0]], [measured[0]], 'rx', - [ideal[1]], [measured[1]], 'gx', - [ideal[2]], [measured[2]], 'bx') - - fits = [fit_fn(ideals[n], measureds[n]) for n in [0, 1, 2]] - pyplot.plot(range(0, 256), [fits[0](x) for x in range(0, 256)], 'r-', - range(0, 256), [fits[1](x) for x in range(0, 256)], 'g-', - range(0, 256), [fits[2](x) for x in range(0, 256)], 'b-') - pyplot.draw() - - try: - yield update - finally: - pyplot.close() - - -def _can_show_graphs(): - try: - # pylint:disable=unused-variable - from matplotlib import pyplot - from scipy.optimize import curve_fit - from scipy.interpolate import interp1d - return True - except ImportError: - sys.stderr.write("Install matplotlib and scipy for graphical " - "assistance with colour calibration\n") - return False - - -def adjust_levels(dut, tv, device): - tv.show("colours2") - with colour_graph(dut) as update_graph: - update_graph() - while prompt_for_adjustment(device): - update_graph() - - -# -# Uniform Illumination -# - -FRAME_AVERAGE_COUNT = 16 - -videos['blank-white'] = ( - 'video/x-raw,format=BGR,width=1280,height=720', - lambda: [(bytearray([0xff, 0xff, 0xff]) * 1280 * 720, 60 * Gst.SECOND)]) -videos['blank-black'] = ( - 'video/x-raw,format=BGR,width=1280,height=720', - lambda: [(bytearray([0, 0, 0]) * 1280 * 720, 60 * Gst.SECOND)]) - - -def _create_reference_png(dut, filename): - # Throw away some frames to let everything settle - pop_with_progress(dut.frames(), 50) - - average = None - for frame in pop_with_progress(dut.frames(), FRAME_AVERAGE_COUNT): - if average is None: - average = numpy.zeros(shape=frame.shape, dtype=numpy.uint16) - average += frame - average /= FRAME_AVERAGE_COUNT - cv2.imwrite(filename, numpy.array(average, dtype=numpy.uint8)) - - -def await_blank(dut, brightness): - for frame in dut.frames(10): - grayscale = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - min_, max_, _, _ = cv2.minMaxLoc(grayscale) - contrast = max_ - min_ - if contrast < 100 and abs(numpy.median(frame) - brightness) < 100: - break - else: - sys.stderr.write( - "WARNING: Did not detect blank frame of brightness %i" % brightness) - - -def calibrate_illumination(dut, tv): - img_dir = xdg_config_dir() + '/stbt/' - - props = { - 'white-reference-image': '%s/vignetting-reference-white.png' % img_dir, - 'black-reference-image': '%s/vignetting-reference-black.png' % img_dir, - } - - tv.show("blank-white") - await_blank(dut, 255) - _create_reference_png(dut, props['white-reference-image']) - tv.show("blank-black") - await_blank(dut, 0) - _create_reference_png(dut, props['black-reference-image']) - - contraststretch = dut._display.source_pipeline.get_by_name( - 'illumination_correction') - for k, v in reversed(props.items()): - contraststretch.set_property(k, v) - set_config( - 'global', 'contraststretch_params', - ' '.join(["%s=%s" % (k, v) for k, v in props.items()])) - - -# -# setup -# - -uvcvideosrc = ('uvch264src device=%(v4l2_device)s name=src auto-start=true ' - 'rate-control=vbr initial-bitrate=5000000 ' - 'peak-bitrate=10000000 average-bitrate=5000000 ' - 'v4l2src0::extra-controls="ctrls, %(v4l2_ctls)s" src.vidsrc ! ' - 'video/x-h264,width=1920 ! h264parse') -v4l2videosrc = 'v4l2src device=%(v4l2_device)s extra-controls=%(v4l2_ctls)s' - - -def list_cameras(): - gi.require_version('GUdev', '1.0') - from gi.repository import GUdev - client = GUdev.Client.new(['video4linux/usb_device']) - devices = client.query_by_subsystem('video4linux') - for d in devices: - # Prefer to refer to a device by path. This means that you are - # referring to a particular USB port and is stable across reboots. - dev_files = d.get_device_file_symlinks() - path_dev_files = [x for x in dev_files if 'by-path' in x] - dev_file = (path_dev_files + [d.get_device_file])[0] - - name = (d.get_property('ID_VENDOR_ENC').decode('string-escape') + ' ' + - d.get_property('ID_MODEL_ENC').decode('string-escape')) - - if d.get_property('ID_USB_DRIVER') == 'uvcvideo': - source_pipeline = uvcvideosrc - else: - source_pipeline = v4l2videosrc - - yield (name, dev_file, source_pipeline) - - -def setup(source_pipeline): - """If we haven't got a configured camera offer a list of cameras you might - want to use. In the future it could be useful to allow the user to select - one from the list interactively.""" - if (source_pipeline == '' or - stbt.get_config('global', 'v4l2_device', '') == ''): - sys.stderr.write( - 'No camera configured in stbt.conf please add parameters ' - '"v4l2_device" and "source_pipeline" to section [global] of ' - 'stbt.conf.\n\n') - cameras = list(list_cameras()) - if len(cameras) == 0: - sys.stderr.write("No Cameras Detected\n\n") - else: - sys.stderr.write("Detected cameras:\n\n") - for n, (name, dev_file, pipeline) in enumerate(cameras): - sys.stderr.write( - " %i. %s\n" - "\n" - " v4l2_device = %s\n" - " source_pipeline = %s\n\n" - % (n, name, dev_file, pipeline)) - return None - return stbt.get_config('global', 'v4l2_device') - -# -# main -# - -defaults = { - 'contraststretch_params': '', - 'v4l2_ctls': ( - 'brightness=128,contrast=128,saturation=128,' - 'white_balance_temperature_auto=0,white_balance_temperature=6500,' - 'gain=60,backlight_compensation=0,exposure_auto=1,' - 'exposure_absolute=152,focus_auto=0,focus_absolute=0,' - 'power_line_frequency=1'), - 'transformation_pipeline': ( - 'stbtgeometriccorrection name=geometric_correction ' - ' %(geometriccorrection_params)s ' - ' ! stbtcontraststretch name=illumination_correction ' - ' %(contraststretch_params)s '), -} - - -def parse_args(argv): - parser = _stbt.core.argparser() - tv_driver.add_argparse_argument(parser) - parser.add_argument( - '--noninteractive', action="store_false", dest="interactive", - help="Don't prompt, assume default answer to all questions") - parser.add_argument( - '--skip-geometric', action="store_true", - help="Don't perform geometric calibration") - parser.add_argument( - '--skip-illumination', action='store_true', - help="Don't perform uniform illumination calibration") - return parser.parse_args(argv[1:]) - - -def main(argv): - args = parse_args(argv) - - device = setup(args.source_pipeline) - if device is None: - return 1 - - if args.skip_geometric: - set_config('global', 'geometriccorrection_params', '') - - for k, v in defaults.iteritems(): - set_config('global', k, v) - - # Need to re-parse arguments as the settings above may have affected the - # values we get out. - args = parse_args(argv) - - transformation_pipeline = ( - 'tee name=raw_undistorted ' - 'raw_undistorted. ! queue leaky=upstream ! videoconvert ! ' - ' textoverlay text="Capture from camera" ! %s ' - 'raw_undistorted. ! queue ! appsink drop=true sync=false qos=false' - ' max-buffers=1 caps="video/x-raw,format=BGR"' - ' name=undistorted_appsink ' - 'raw_undistorted. ! queue leaky=upstream max-size-buffers=1 ! %s' % - (args.sink_pipeline, - stbt.get_config('global', 'transformation_pipeline'))) - - args.sink_pipeline = ('textoverlay text="After correction" ! ' + - args.sink_pipeline) - args.control = 'none' - - with _stbt.core.new_device_under_test_from_config( - args, transformation_pipeline=transformation_pipeline) as dut: - tv = tv_driver.create_from_args(args, videos) - - if not args.skip_geometric: - geometric_calibration(dut, tv, device, interactive=args.interactive) - if args.interactive: - adjust_levels(dut, tv, device) - if not args.skip_illumination: - calibrate_illumination(dut, tv) - - if args.interactive: - raw_input("Calibration complete. Press to exit") - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-camera.d/stbt_camera_validate.py stb-tester-31/stbt-camera.d/stbt_camera_validate.py --- stb-tester-30-5-gbefe47c/stbt-camera.d/stbt_camera_validate.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-camera.d/stbt_camera_validate.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,241 +0,0 @@ -#!/usr/bin/python -u -# coding: utf-8 -from __future__ import unicode_literals - -import argparse -import math -import os -import sys -from collections import namedtuple -from os.path import abspath, dirname - -import gi -import numpy - -from _stbt import tv_driver - -gi.require_version("Gst", "1.0") -from gi.repository import Gst # pylint:disable=wrong-import-order - -DATA_DIR = dirname(abspath(__file__)) - -STANDARD_COLOURS = { - 'letters-bw': ('#ffffff', '#000000'), - 'letters-wb': ('#000000', '#ffffff'), - 'letters-grey': ('#AAAAAA', '#555555'), - 'letters-grey-inv': ('#555555', '#AAAAAA'), -} - -Coord = namedtuple('Coord', 'x y') - -SQUARES = [Coord(_x, _y) for _y in range(0, 9) for _x in range(0, 16)] -GLYPHS = list( - 'ABCDEFGHIJKLMNOP' + - 'QRSTUVWXYZ012345' + - '6789abcdefghijkm' + - 'nopqrstuvwxyzΓΔΘ' + - 'ΛΞΠΣΦΨΩαβγδεζηθι' + - 'κλμνξπρςστυφχψωб' + - 'джзйлптфцчшщъыьэ' + - 'юя@#~!£$%+-¶()?[' + - ']¿÷«»©®℠℗™&<>^/*') - - -def square_to_pos(square): - return Coord(square[0] * 80, square[1] * 80) - - -def distance(a, b): - return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2) ** 0.5 - - -def off_to_arrow(off): - """ - >>> print off_to_arrow((1, 1)) - ↘ - >>> print off_to_arrow((-1, 0)) - ← - """ - arrows = list('←↙↓↘→↗↑↖') - if numpy.linalg.norm(off) > 0.5: - angle = math.atan2(off[1], -off[0]) - return arrows[int(angle / 2 / math.pi * len(arrows) + len(arrows) + 0.5) - % len(arrows)] - else: - return 'O' - - -# ANSI colour codes for printing progress. -OKGREEN = '\033[92m' -WARNING = '\033[93m' -FAIL = '\033[91m' -ENDC = '\033[0m' - - -def rate(square, result): - """How good is the match on a scale of 0-2?""" - r = distance(square_to_pos(square), result.position) - if r < 0.5: - return 2 - elif r < 20: - return 1 - else: - return 0 - - -def length(vec): - return (vec[0] ** 2 + vec[1] ** 2) ** 0.5 - - -def svg_to_array(svg): - from _stbt.gst_utils import array_from_sample - pipeline = Gst.parse_launch( - 'appsrc name="src" caps="image/svg" ! rsvgdec ! ' - 'videoconvert ! appsink caps="video/x-raw,format=BGR" name="sink"') - src = pipeline.get_by_name('src') - sink = pipeline.get_by_name('sink') - pipeline.set_state(Gst.State.PLAYING) - buf = Gst.Buffer.new_wrapped(svg) - src.emit('push-buffer', buf) - sample = sink.emit('pull-sample') - src.emit("end-of-stream") - pipeline.set_state(Gst.State.NULL) - pipeline.get_state(0) - return array_from_sample(sample) - - -def generate_letters_svg(fgcolour, bgcolour): - from jinja2 import Template - - posns = [(x, y) - for y in range(60, 720 + 60, 80) - for x in range(40, 1280 + 40, 80)] - data = [{'glyph': g, 'x': x, 'y': y} for g, (x, y) in zip(GLYPHS, posns)] - - svg = (Template(open(DATA_DIR + '/glyphs.svg.jinja2', 'r').read()) - .render(glyphs=data, fgcolour=fgcolour, bgcolour=bgcolour) - .encode('utf-8')) - return svg - - -videos = { - name: - ('image/svg', - lambda bg=bg, fg=fg: [(generate_letters_svg(fg, bg), 240 * Gst.SECOND)]) - for name, (fg, bg) in STANDARD_COLOURS.items()} - - -def validate(video, driver, validate_match=True): - import stbt - - driver.show(video) - - colours = STANDARD_COLOURS[video] - pristine = svg_to_array(generate_letters_svg(*colours)) - - # Attempt to wait until correct video is showing. - try: - # environment variable controls the timeout here as with the - # fake-video-source with the parallel test runner 10s can not be enough - timeout = int(os.environ.get('STBT_TEST_VALIDATION_WAIT_TIMEOUT', 10)) - stbt.wait_for_match(pristine[80 * 3:80 * 6, 80 * 6:80 * 10], - timeout_secs=timeout) - except stbt.MatchTimeout: - pass - - res = [] - for square, char in zip(SQUARES, GLYPHS): - pos = square_to_pos(square) - template = pristine[pos.y:pos.y + 80, pos.x:pos.x + 80] - result = stbt.match( - template, - match_parameters=stbt.MatchParameters(match_method="ccoeff-normed")) - sys.stdout.write( - "%s%s" % ([FAIL, WARNING, OKGREEN][rate(square, result)], char)) - if square.x == 15: - sys.stdout.write('\n') - sys.stdout.flush() - res.append((square, char, result)) - sys.stdout.write(ENDC) - - sys.stdout.write('\n\n') - for square, char, result in res: - expected = square_to_pos(square) - off = Coord(result.position[0] - expected.x, - result.position[1] - expected.y) - sys.stdout.write( - "%s%s" % ([FAIL, WARNING, OKGREEN][rate(square, result)], - off_to_arrow(off))) - if square.x == 15: - sys.stdout.write('\n') - sys.stdout.write(ENDC) - - if validate_match: - sys.stdout.write('\n\n') - for square, char, result in res: - if result.match: - rating = 2 - else: - rating = 1 if result.first_pass_result > 0.9 else 0 - quality = "01234567899"[int(result.first_pass_result * 10)] - sys.stdout.write( - "%s%s" % ([FAIL, WARNING, OKGREEN][rating], quality)) - if square.x == 15: - sys.stdout.write('\n') - sys.stdout.write(ENDC) - - is_bad = 0 - for square, char, result in res: - good = rate(square, result) == 2 - if validate_match: - good = good and result.match - - if not good: - if is_bad == 0: - is_bad = 1 - sys.stdout.write('\nChar\tPos\tOffset\tDist\tMatch\n' + - '-' * 40 + '\n') - expected = square_to_pos(square) - off = Coord(result.position[0] - expected.x, - result.position[1] - expected.y) - sys.stdout.write('%s%s\t(%i, %i)\t(%i, %i)\t%02f\t%s\n' % - ([FAIL, WARNING, OKGREEN][rate(square, result)], - char, square.x, square.y, off.x, off.y, - distance(expected, result.position), - "MATCH" if result.match else "NO MATCH")) - sys.stdout.write(ENDC) - sys.stdout.flush() - return is_bad - - -def main(argv): - parser = argparse.ArgumentParser() - - parser.add_argument( - "--report-style", choices=['text', 'color-text', 'html'], - help='Style of report to use', - default='color-text') - parser.add_argument( - '--positions-only', action="store_true", - help="Only validate that the letters are in their right positions. " - "Don't require that they match") - parser.add_argument( - '-o', '--output', type=argparse.FileType('w'), - help='Filename to write the report to') - tv_driver.add_argparse_argument(parser) - - parser.add_argument("colour", nargs='*') - - args = parser.parse_args(argv[1:]) - - if len(args.colour) == 0: - args.colour = STANDARD_COLOURS.keys() - - driver = tv_driver.create_from_args(args, videos) - failures = 0 - for c in args.colour: - failures += validate(c, driver, validate_match=not args.positions_only) - return failures - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-completion stb-tester-31/stbt-completion --- stb-tester-30-5-gbefe47c/stbt-completion 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-completion 2019-09-18 14:04:32.000000000 +0000 @@ -6,8 +6,8 @@ # https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). complete -o nospace -F _stbt stbt -complete -o nospace -F _stbt_run stbt-run -complete -o nospace -F _stbt_record stbt-record +complete -o nospace -F _stbt_run stbt_run.py +complete -o nospace -F _stbt_record stbt_record.py _stbt() { local cur="${COMP_WORDS[COMP_CWORD]}" @@ -15,8 +15,6 @@ if [ $COMP_CWORD = 1 ]; then COMPREPLY=($(compgen \ -W "$(_stbt_trailing_space --help --version \ - auto-selftest \ - batch \ config \ control \ lint \ @@ -29,8 +27,6 @@ -- "$cur")) else case "${COMP_WORDS[1]}" in - auto-selftest) _stbt_auto_selftest;; - batch) _stbt_batch;; config) _stbt_config;; control) _stbt_control;; lint) _stbt_lint;; @@ -54,7 +50,7 @@ --save-video=*) COMPREPLY=($(_stbt_filenames "$cur"));; *) COMPREPLY=( $(compgen -W "$(_stbt_trailing_space \ - --help --verbose --restart-source --save-video \ + --help --verbose --save-video \ --control --source-pipeline --sink-pipeline)" \ -- "$cur") $(_stbt_filename_possibly_with_test_functions));; @@ -73,51 +69,13 @@ -o=*|--output-file=*) COMPREPLY=($(_stbt_filenames "$cur"));; *) COMPREPLY=($(compgen \ -W "$(_stbt_trailing_space \ - --help --verbose --restart-source \ + --help --verbose \ --control --source-pipeline --sink-pipeline \ --control-recorder --output-file)" \ -- "$cur"));; esac } -_stbt_auto_selftest() { - local cur="${COMP_WORDS[COMP_CWORD]}" - local prev="${COMP_WORDS[COMP_CWORD-1]}" - local subcommand="${COMP_WORDS[2]}" - case "$prev" in - auto-selftest) - COMPREPLY=($(compgen \ - -W "$(_stbt_trailing_space --help generate validate)" \ - -- "$cur")); - return;; - esac - case "$subcommand" in - --help|validate) COMPREPLY=();; - generate) COMPREPLY=($(_stbt_filenames "$cur"));; - esac -} - -_stbt_batch() { - _stbt_get_prev - local cur="${COMP_WORDS[COMP_CWORD]}" - local prev="${COMP_WORDS[COMP_CWORD-1]}" - local subcommand="${COMP_WORDS[2]}" - case "$COMP_CWORD,$subcommand" in - 2,*) COMPREPLY=($(compgen \ - -W "$(_stbt_trailing_space --help run report instaweb)" \ - -- "$cur"));; - 3,run) COMPREPLY=( - $(compgen -W "-h" -- "$cur") - $(_stbt_filename_possibly_with_test_functions));; - *,run) COMPREPLY=($(_stbt_filename_possibly_with_test_functions));; - *,report) COMPREPLY=($(compgen -d -S "/" -- "$cur"));; - 3,instaweb) COMPREPLY=($(compgen \ - -W "$(_stbt_trailing_space \ - --help localhost:5000 0.0.0.0:5000)" \ - -- "$cur"));; - esac -} - _stbt_config() { local cur="${COMP_WORDS[COMP_CWORD]}" local prev="${COMP_WORDS[COMP_CWORD-1]}" diff -Nru stb-tester-30-5-gbefe47c/stbt-config stb-tester-31/stbt-config --- stb-tester-30-5-gbefe47c/stbt-config 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-config 1970-01-01 00:00:00.000000000 +0000 @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -""" -Copyright 2013 YouView TV Ltd. -License: LGPL v2.1 or (at your option) any later version (see -https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). -""" - -import argparse -import sys - -from _stbt.config import _config_init, ConfigurationError, get_config - - -def error(s): - sys.stderr.write("stbt config: error: %s\n" % s) - sys.exit(1) - - -parser = argparse.ArgumentParser() -parser.prog = "stbt config" -parser.description = """Prints the value of the specified key from the stbt - configuration file. See 'configuration' in the stbt(1) man page.""" -parser.epilog = \ - "Returns non-zero exit status if the specified key or section isn't found." -parser.add_argument( - "--bash-completion", action="store_true", help=argparse.SUPPRESS) -parser.add_argument( - "name", metavar="section.key", - help="e.g. 'global.source_pipeline' or 'record.control_recorder'") -args = parser.parse_args(sys.argv[1:]) - -if args.bash_completion: - cfg = _config_init() - for section in cfg.sections(): - for option in cfg.options(section): - print "%s.%s" % (section, option) - sys.exit(0) - -if args.name.rfind(".") == -1: - error( - "'name' parameter must contain the section and key separated by a dot") - -section, key = args.name.rsplit(".", 1) - -try: - print get_config(section, key) -except ConfigurationError as e: - error(e.message) diff -Nru stb-tester-30-5-gbefe47c/stbt_config.py stb-tester-31/stbt_config.py --- stb-tester-30-5-gbefe47c/stbt_config.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_config.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,59 @@ +#!/usr/bin/python + +""" +Copyright 2013 YouView TV Ltd. +License: LGPL v2.1 or (at your option) any later version (see +https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). +""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import argparse +import sys + +from _stbt.config import _config_init, ConfigurationError, get_config + + +def error(s): + sys.stderr.write("stbt config: error: %s\n" % s) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.prog = "stbt config" + parser.description = """Prints the value of the specified key from the stbt + configuration file. See 'configuration' in the stbt(1) man page.""" + parser.epilog = """Returns non-zero exit status if the specified key or + section isn't found.""" + parser.add_argument( + "--bash-completion", action="store_true", help=argparse.SUPPRESS) + parser.add_argument( + "name", metavar="section.key", + help="e.g. 'global.source_pipeline' or 'record.control_recorder'") + args = parser.parse_args(sys.argv[1:]) + + if args.bash_completion: + cfg = _config_init() + for section in cfg.sections(): + for option in cfg.options(section): + print("%s.%s" % (section, option)) + sys.exit(0) + + if args.name.rfind(".") == -1: + error("'name' parameter must contain the section and key " + "separated by a dot") + + section, key = args.name.rsplit(".", 1) + + try: + print(get_config(section, key)) + except ConfigurationError as e: + error(e) + + +if __name__ == "__main__": + main() diff -Nru stb-tester-30-5-gbefe47c/stbt-control stb-tester-31/stbt-control --- stb-tester-30-5-gbefe47c/stbt-control 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-control 1970-01-01 00:00:00.000000000 +0000 @@ -1,294 +0,0 @@ -#!/usr/bin/env python -"""Send remote control signals using the PC keyboard or from the command line. -""" - -import argparse -import collections -import contextlib -import curses -import curses.ascii -import math -import os -import sys -import threading -import time -from textwrap import dedent - -import _stbt.control -from _stbt.config import ConfigurationError, get_config - -SPECIAL_CHARS = { - curses.ascii.SP: "Space", - curses.ascii.NL: "Enter", - curses.ascii.TAB: "Tab", - curses.ascii.ESC: "Escape", - curses.KEY_UP: "Up", - curses.KEY_DOWN: "Down", - curses.KEY_LEFT: "Left", - curses.KEY_RIGHT: "Right", - curses.KEY_BACKSPACE: "Backspace", - curses.KEY_PPAGE: "PageUp", - curses.KEY_NPAGE: "PageDown", - curses.KEY_HOME: "Home", - curses.KEY_END: "End", - curses.KEY_IC: "Insert", - curses.KEY_DC: "Delete", -} - - -def main(argv): - args = argparser().parse_args(argv[1:]) - - if args.help_keymap: - sys.exit(show_help_keymap()) - - control = _stbt.control.uri_to_control(args.control, None) - - if args.remote_control_key: # Send a single key and exit - control.press(args.remote_control_key) - else: # Interactive - for key in main_loop(args.control, args.keymap): - control.press(key) - - -def argparser(): - parser = argparse.ArgumentParser() - parser.prog = "stbt control" - parser.description = ("Send remote control signals using the PC keyboard " - "or from the command line.") - parser.add_argument( - "--help-keymap", action='store_true', default=False, - help="Show description of the keymap file format and exit.") - parser.add_argument( - "--keymap", default=default_keymap_file(), - help="Load keymap from KEYMAP file; defaults to %(default)s. " - "See `%(prog)s --help-keymap` for details.") - parser.add_argument( - "--control", default=get_config("global", "control"), - help="Equivalent to the --control parameter of `stbt run`. " - "See `man stbt` for available control types and configuration.") - parser.add_argument( - "remote_control_key", default=None, nargs='?', - help=( - "The name of a remote control key as in the control's config file " - "(that is /etc/lirc/lircd.conf in case of a LIRC control device). " - "Specifying this argument sends remote_control_key and exits. " - "Omitting this argument brings up the printed keymap.")) - return parser - - -def show_help_keymap(): - """Keymap File - - A keymap file stores the mappings between keyboard keys and remote control - keys. One line of the file stores one key mapping in the following format: - - [] - - is an ASCII character or one of the following keywords: - - Space, Enter, Tab, Escape, Up, Down, Left, Right, Backspace, - PageUp, PageDown, Home, End, Insert, Delete - - Be careful that keywords are case sensitive. - - is the same as in the command line arguments. It - cannot contain white spaces. - - is an optional alias for to show in the - on-screen keymap; e.g. "m MENU Main Menu" displays "Main Menu" but sends - the "MENU" remote control key when "m" is pressed on the PC keyboard. It - may consist of multiple words but cannot be longer than 15 characters. - - Comments start with '//' (double slash) and last until the end of line. - - Example keymap: - - m MENU Main Menu - Enter OK - c CLOSE Close // Go back to live TV - - The keymap file to use can be specified by: - - 1. Passing the filename of the keymap to the --keymap= command - line argument. - 2. Or setting the configuration value `control.keymap` to the filename of - the keymap file - 3. Copying it into $XDG_CONFIG_PATH/stbt/control.conf (defaults to - $HOME/.config/stbt/control.conf). - """ - print globals()["show_help_keymap"].__doc__ - - -def main_loop(control_uri, keymap_file): - try: - keymap = load_keymap(open(keymap_file, "r")) - except IOError: - error("Failed to load keymap file '%s'\n" - "(see 'stbt control --help' for details of the keymap file)." - % keymap_file) - timer = None - - with terminal() as term: - _, term_width = term.getmaxyx() - printed_keymap = dedent("""\ - Keymap: {keymap_file} - Control: {control_uri} - - {mapping} - - """).format(keymap_file=keymap_file[:term_width - len("Keymap: ")], - control_uri=control_uri[:77] + ( - control_uri[77:] and "..."), - mapping=keymap_string(keymap)) - if keymap_fits_terminal(term, printed_keymap): - term.addstr(printed_keymap) - else: - raise EnvironmentError( - "Unable to print keymap because the terminal is too small. " - "Please resize the terminal window.") - while True: # Main loop - keycode = term.getch() - if keycode == ord('q'): # 'q' for 'Quit' is pre-defined - return - - control_key, _ = keymap.get(decoded(keycode), (None, None)) - if timer: - timer.cancel() - clear_last_command(term) - if control_key: - yield control_key - term.addstr(str(control_key)) - timer = threading.Timer(1, clear_last_command, [term]) - timer.start() - time.sleep(.2) - curses.flushinp() - - -def clear_last_command(term): - term.move(term.getyx()[0], 0) - term.clrtoeol() - - -def keymap_fits_terminal(term, printed_keymap): - term_y, term_x = term.getmaxyx() - lines = printed_keymap.split("\n") - return term_y > len(lines) and term_x > max(len(line) for line in lines) - - -@contextlib.contextmanager -def terminal(): - term = curses.initscr() - curses.noecho() - curses.cbreak() - term.keypad(1) - term.immedok(1) - try: - yield term - finally: - term.immedok(0) - term.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - - -def keymap_string(keymap): - """ - >>> print keymap_string({"m": ("MENU", "Main Menu")}).strip() - q - m - Main Menu - """ - keylist = ["%15s - %-15s" % (kb_key, mapping[1]) - for kb_key, mapping in keymap.items()] - keylist.insert(0, "%15s - %-15s" % ("q", "")) - middle = int(math.ceil(float(len(keylist)) / 2)) - rows = [ - "%s %s" % ( - keylist[i], - keylist[middle + i] if middle + i < len(keylist) else "") - for i in range(middle)] - return "\n".join(rows) - - -def decoded(keycode): - """ - >>> decoded(curses.KEY_BACKSPACE) - 'Backspace' - >>> decoded(120) - 'x' - >>> decoded(curses.KEY_F12) - """ - if keycode in SPECIAL_CHARS.keys(): - return SPECIAL_CHARS[keycode] - try: - return chr(keycode) - except ValueError: - return None - - -def load_keymap(keymap_file): - keymap = collections.OrderedDict() - for line in keymap_file: - items = line.split("//")[0].split() - if len(items) < 2: - continue - if items[0] in keymap: - raise ConfigurationError( - "Multiple remote control keys assigned to keyboard key '%s' " - "in the keymap file" % items[0]) - if len(items) == 2: - keymap[items[0]] = (items[1],) * 2 - else: - keymap[items[0]] = (items[1], " ".join(items[2:])) - validate(items[0]) - return keymap - - -def test_load_keymap(): - keymap = load_keymap(__import__("StringIO").StringIO( - "Backspace BACK Go back to previous\n" # Test display name - "Enter OK //Move forward\n")) # Test comments - assert keymap["Backspace"] == ("BACK", "Go back to previous") - assert keymap["Enter"] == ("OK", "OK") - - -def test_load_keymap_with_duplicated_key(): - try: - load_keymap(__import__("StringIO").StringIO("o OK\no OPEN\n")) - assert False, "load_keymap should have thrown ConfigurationError" - except ConfigurationError: - pass - - -def validate(keyname): - if keyname in SPECIAL_CHARS.values(): - return - try: - ord(keyname) - except TypeError: - raise ValueError("Invalid keyboard key in the keymap file: " + keyname) - - -def test_validate(): - try: - validate("Invalid") - assert False - except ValueError: - pass - - -def default_keymap_file(): - config_dir = os.environ.get( - 'XDG_CONFIG_HOME', '%s/.config' % os.environ['HOME']) - return get_config("control", "keymap", "") or \ - os.path.join(config_dir, "stbt", "control.conf") - - -def error(s): - sys.stderr.write("%s: error: %s\n" % ( - os.path.basename(sys.argv[0]), str(s))) - sys.exit(1) - - -if __name__ == "__main__": - main(sys.argv) diff -Nru stb-tester-30-5-gbefe47c/stbt_control.py stb-tester-31/stbt_control.py --- stb-tester-30-5-gbefe47c/stbt_control.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_control.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,299 @@ +#!/usr/bin/python +"""Send remote control signals using the PC keyboard or from the command line. +""" +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import argparse +import collections +import contextlib +import curses +import curses.ascii +import io +import math +import os +import sys +import threading +import time +from textwrap import dedent + +import _stbt.control +from _stbt.config import ConfigurationError, get_config + +SPECIAL_CHARS = { + curses.ascii.SP: "Space", + curses.ascii.NL: "Enter", + curses.ascii.TAB: "Tab", + curses.ascii.ESC: "Escape", + curses.KEY_UP: "Up", + curses.KEY_DOWN: "Down", + curses.KEY_LEFT: "Left", + curses.KEY_RIGHT: "Right", + curses.KEY_BACKSPACE: "Backspace", + curses.KEY_PPAGE: "PageUp", + curses.KEY_NPAGE: "PageDown", + curses.KEY_HOME: "Home", + curses.KEY_END: "End", + curses.KEY_IC: "Insert", + curses.KEY_DC: "Delete", +} + + +def main(argv): + args = argparser().parse_args(argv[1:]) + + if args.help_keymap: + sys.exit(show_help_keymap()) + + control = _stbt.control.uri_to_control(args.control, None) + + if args.remote_control_key: # Send a single key and exit + control.press(args.remote_control_key) + else: # Interactive + for key in main_loop(args.control, args.keymap): + control.press(key) + + +def argparser(): + parser = argparse.ArgumentParser() + parser.prog = "stbt control" + parser.description = ("Send remote control signals using the PC keyboard " + "or from the command line.") + parser.add_argument( + "--help-keymap", action='store_true', default=False, + help="Show description of the keymap file format and exit.") + parser.add_argument( + "--keymap", default=default_keymap_file(), + help="Load keymap from KEYMAP file; defaults to %(default)s. " + "See `%(prog)s --help-keymap` for details.") + parser.add_argument( + "--control", default=get_config("global", "control"), + help="Equivalent to the --control parameter of `stbt run`. " + "See `man stbt` for available control types and configuration.") + parser.add_argument( + "remote_control_key", default=None, nargs='?', + help=( + "The name of a remote control key as in the control's config file " + "(that is /etc/lirc/lircd.conf in case of a LIRC control device). " + "Specifying this argument sends remote_control_key and exits. " + "Omitting this argument brings up the printed keymap.")) + return parser + + +def show_help_keymap(): + """Keymap File + + A keymap file stores the mappings between keyboard keys and remote control + keys. One line of the file stores one key mapping in the following format: + + [] + + is an ASCII character or one of the following keywords: + + Space, Enter, Tab, Escape, Up, Down, Left, Right, Backspace, + PageUp, PageDown, Home, End, Insert, Delete + + Be careful that keywords are case sensitive. + + is the same as in the command line arguments. It + cannot contain white spaces. + + is an optional alias for to show in the + on-screen keymap; e.g. "m MENU Main Menu" displays "Main Menu" but sends + the "MENU" remote control key when "m" is pressed on the PC keyboard. It + may consist of multiple words but cannot be longer than 15 characters. + + Comments start with '//' (double slash) and last until the end of line. + + Example keymap: + + m MENU Main Menu + Enter OK + c CLOSE Close // Go back to live TV + + The keymap file to use can be specified by: + + 1. Passing the filename of the keymap to the --keymap= command + line argument. + 2. Or setting the configuration value `control.keymap` to the filename of + the keymap file + 3. Copying it into $XDG_CONFIG_PATH/stbt/control.conf (defaults to + $HOME/.config/stbt/control.conf). + """ + print(globals()["show_help_keymap"].__doc__) + + +def main_loop(control_uri, keymap_file): + try: + keymap = load_keymap(open(keymap_file, "r")) + except IOError: + error("Failed to load keymap file '%s'\n" + "(see 'stbt control --help' for details of the keymap file)." + % keymap_file) + timer = None + + with terminal() as term: + _, term_width = term.getmaxyx() + printed_keymap = dedent("""\ + Keymap: {keymap_file} + Control: {control_uri} + + {mapping} + + """).format(keymap_file=keymap_file[:term_width - len("Keymap: ")], + control_uri=control_uri[:77] + ( + control_uri[77:] and "..."), + mapping=keymap_string(keymap)) + if keymap_fits_terminal(term, printed_keymap): + term.addstr(printed_keymap) + else: + raise EnvironmentError( + "Unable to print keymap because the terminal is too small. " + "Please resize the terminal window.") + while True: # Main loop + keycode = term.getch() + if keycode == ord('q'): # 'q' for 'Quit' is pre-defined + return + + control_key, _ = keymap.get(decoded(keycode), (None, None)) + if timer: + timer.cancel() + clear_last_command(term) + if control_key: + yield control_key + term.addstr(str(control_key)) + timer = threading.Timer(1, clear_last_command, [term]) + timer.start() + time.sleep(.2) + curses.flushinp() + + +def clear_last_command(term): + term.move(term.getyx()[0], 0) + term.clrtoeol() + + +def keymap_fits_terminal(term, printed_keymap): + term_y, term_x = term.getmaxyx() + lines = printed_keymap.split("\n") + return term_y > len(lines) and term_x > max(len(line) for line in lines) + + +@contextlib.contextmanager +def terminal(): + term = curses.initscr() + curses.noecho() + curses.cbreak() + term.keypad(1) + term.immedok(1) + try: + yield term + finally: + term.immedok(0) + term.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + +def keymap_string(keymap): + """ + >>> print(keymap_string({"m": ("MENU", "Main Menu")}).strip()) + q - m - Main Menu + """ + keylist = ["%15s - %-15s" % (kb_key, mapping[1]) + for kb_key, mapping in keymap.items()] + keylist.insert(0, "%15s - %-15s" % ("q", "")) + middle = int(math.ceil(float(len(keylist)) / 2)) + rows = [ + "%s %s" % ( + keylist[i], + keylist[middle + i] if middle + i < len(keylist) else "") + for i in range(middle)] + return "\n".join(rows) + + +def decoded(keycode): + """ + >>> decoded(curses.KEY_BACKSPACE) + 'Backspace' + >>> decoded(120) + 'x' + """ + if keycode in SPECIAL_CHARS: + return SPECIAL_CHARS[keycode] + try: + return chr(keycode) + except ValueError: + return None + + +def load_keymap(keymap_file): + keymap = collections.OrderedDict() + for line in keymap_file: + items = line.split("//")[0].split() + if len(items) < 2: + continue + if items[0] in keymap: + raise ConfigurationError( + "Multiple remote control keys assigned to keyboard key '%s' " + "in the keymap file" % items[0]) + if len(items) == 2: + keymap[items[0]] = (items[1],) * 2 + else: + keymap[items[0]] = (items[1], " ".join(items[2:])) + validate(items[0]) + return keymap + + +def test_load_keymap(): + keymap = load_keymap(io.StringIO( + "Backspace BACK Go back to previous\n" # Test display name + "Enter OK //Move forward\n")) # Test comments + assert keymap["Backspace"] == ("BACK", "Go back to previous") + assert keymap["Enter"] == ("OK", "OK") + + +def test_load_keymap_with_duplicated_key(): + try: + load_keymap(io.StringIO("o OK\no OPEN\n")) + assert False, "load_keymap should have thrown ConfigurationError" + except ConfigurationError: + pass + + +def validate(keyname): + if keyname in SPECIAL_CHARS.values(): + return + try: + ord(keyname) + except TypeError: + raise ValueError("Invalid keyboard key in the keymap file: " + keyname) + + +def test_validate(): + try: + validate("Invalid") + assert False + except ValueError: + pass + + +def default_keymap_file(): + config_dir = os.environ.get( + 'XDG_CONFIG_HOME', '%s/.config' % os.environ['HOME']) + return get_config("control", "keymap", "") or \ + os.path.join(config_dir, "stbt", "control.conf") + + +def error(s): + sys.stderr.write("%s: error: %s\n" % ( + os.path.basename(sys.argv[0]), str(s))) + sys.exit(1) + + +if __name__ == "__main__": + main(sys.argv) diff -Nru stb-tester-30-5-gbefe47c/stbt_control_relay.py stb-tester-31/stbt_control_relay.py --- stb-tester-30-5-gbefe47c/stbt_control_relay.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt_control_relay.py 2019-09-18 14:04:32.000000000 +0000 @@ -27,15 +27,21 @@ Will press KEY_OK on the roku device. """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import argparse +import logging import os import re import signal import socket import sys -import _stbt.logging from _stbt.control import uri_to_control +from _stbt.utils import to_bytes def main(argv): @@ -44,11 +50,15 @@ parser.add_argument( "--socket", default="/var/run/lirc/lircd", help="""LIRC socket to read remote control presses from (defaults to %(default)s).""") + parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("output", help="""Remote control configuration to transmit on. Values are the same as stbt run's --control.""") - _stbt.logging.argparser_add_verbose_argument(parser) args = parser.parse_args(argv[1:]) + logging.basicConfig( + format="%(levelname)s: %(message)s", + level=logging.DEBUG if args.verbose else logging.INFO) + signal.signal(signal.SIGTERM, lambda _signo, _stack_frame: sys.exit(0)) if os.environ.get('LISTEN_FDS') == '1' and \ @@ -61,49 +71,51 @@ control = uri_to_control(args.output) + logging.info("stbt-control-relay started up with output '%s'", args.output) + while True: conn, _ = s.accept() - f = conn.makefile('r', 0) + f = conn.makefile('rb', 0) while True: cmd = f.readline() if not cmd: break - cmd = cmd.rstrip("\n") - m = re.match(r"(?PSEND_ONCE|SEND_START|SEND_STOP) " - r"(?P\S+) (?P\S+)", cmd) + cmd = cmd.rstrip(b"\n") + m = re.match(br"(?PSEND_ONCE|SEND_START|SEND_STOP) " + br"(?P\S+) (?P\S+)", cmd) if not m: - debug("Invalid command: %s" % cmd) + logging.error("Invalid command: %s", cmd) send_response(conn, cmd, success=False, - data="Invalid command: %s" % cmd) + data=b"Invalid command: %s" % cmd) continue - action = m.groupdict()["action"] - key = m.groupdict()["key"] - debug("Received %s %s" % (action, key)) + action = m.group("action") + key = m.group("key") + logging.debug("Received %s %s", action, key) try: - if action == "SEND_ONCE": + key = key.decode("utf-8") + if action == b"SEND_ONCE": control.press(key) - elif action == "SEND_START": + elif action == b"SEND_START": control.keydown(key) - elif action == "SEND_STOP": + elif action == b"SEND_STOP": control.keyup(key) except Exception as e: # pylint: disable=broad-except - debug("Error pressing key %r: %r" % (key, e)) - send_response(conn, cmd, success=False, data=str(e)) + logging.error("Error pressing key %r: %s", key, e, + exc_info=True) + send_response(conn, cmd, success=False, data=to_bytes(str(e))) continue send_response(conn, cmd, success=True) -def send_response(sock, request, success, data=""): +def send_response(sock, request, success, data=b""): # See http://www.lirc.org/html/lircd.html - message = "BEGIN\n{cmd}\n{status}\n".format( - cmd=request, - status="SUCCESS" if success else "ERROR") + message = b"BEGIN\n%s\n%s\n" % ( + request, + b"SUCCESS" if success else b"ERROR") if data: - data = data.split("\n") - message += "DATA\n{length}\n{data}\n".format( - length=len(data), - data="\n".join(data)) - message += "END\n" + data = data.split(b"\n") + message += b"DATA\n%d\n%s\n" % (len(data), b"\n".join(data)) + message += b"END\n" try: sock.sendall(message) @@ -111,9 +123,5 @@ pass -def debug(s): - _stbt.logging.debug("stbt-control-relay: %s" % s) - - if __name__ == "__main__": sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-lint stb-tester-31/stbt-lint --- stb-tester-30-5-gbefe47c/stbt-lint 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-lint 1970-01-01 00:00:00.000000000 +0000 @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2017 Stb-tester.com Ltd. -# Copyright 2013 YouView TV Ltd. -# License: LGPL v2.1 or (at your option) any later version (see -# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). - -"""Run static analysis over the specified stb-tester python scripts. - -"stbt lint" runs "pylint" with the following additional checkers: - -* E7001: The image path given to "stbt.match" (and similar functions) - does not exist on disk. -* E7002: The return value from is_screen_black, match, match_text, ocr, - press_and_wait, or wait_until isn't used (perhaps you've forgotten to - use "assert"). -* E7003: The argument given to "wait_until" must be a callable (such as - a function or lambda expression). -* E7004: FrameObject properties must always provide "self._frame" as the - "frame" parameter to functions such as "stbt.match". -* E7005: The image path given to "stbt.match" (and similar functions) - exists on disk, but isn't committed to git. - -""" - -import argparse -import os -import re -import subprocess -import sys -import threading - - -def main(argv): - parser = argparse.ArgumentParser( - prog="stbt lint", - usage="stbt lint [--help] [pylint options] filename [filename...]", - description=__doc__, - epilog="Any other command-line arguments are passed through to pylint.", - formatter_class=argparse.RawDescriptionHelpFormatter) - _, pylint_args = parser.parse_known_args(argv[1:]) - - if not pylint_args: - parser.print_usage(sys.stderr) - return 1 - - try: - with open("/dev/null", "w") as devnull: - subprocess.check_call(["pylint", "--help"], - stdout=devnull, stderr=devnull) - except OSError as e: - if e.errno == 2: - sys.stderr.write( - "stbt lint: error: Couldn't find 'pylint' executable\n") - return 1 - - env = os.environ - env["PYTHONPATH"] = os.pathsep.join([ - os.path.dirname(__file__), - env.get("PYTHONPATH", "")]) - - pylint = subprocess.Popen( - ["pylint", "--load-plugins=_stbt.pylint_plugin"] + pylint_args, - env=env, stderr=subprocess.PIPE) - - t = threading.Thread(target=filter_warnings, - args=(pylint.stderr, sys.stderr)) - t.start() - - pylint.wait() - t.join() - return pylint.returncode - - -def filter_warnings(input_, output): - while True: - line = input_.readline() - if not line: - break - if any(re.search(pattern, line) for pattern in WARNINGS): - continue - output.write(line) - - -WARNINGS = [ - # pylint:disable=line-too-long - r"libdc1394 error: Failed to initialize libdc1394", - r"pygobject_register_sinkfunc is deprecated", - r"assertion .G_TYPE_IS_BOXED $boxed_type$. failed", - r"assertion .G_IS_PARAM_SPEC $pspec$. failed", - r"return isinstance$object, \(type, types.ClassType$\)", - r"gsignal.c:.*: parameter 1 of type '' for signal \".*\" is not a value type", - r"astroid.* Use gi.require_version", - r"^ __import__$m$$", -] - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt_lint.py stb-tester-31/stbt_lint.py --- stb-tester-30-5-gbefe47c/stbt_lint.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_lint.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,113 @@ +#!/usr/bin/python + +# Copyright 2014-2017 Stb-tester.com Ltd. +# Copyright 2013 YouView TV Ltd. +# License: LGPL v2.1 or (at your option) any later version (see +# https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). + +"""Run static analysis over the specified stb-tester python scripts. + +"stbt lint" runs "pylint" with the following additional checkers: + +* E7001: The image path given to "stbt.match" (and similar functions) + does not exist on disk. +* E7002: The return value from is_screen_black, match, match_text, ocr, + press_and_wait, or wait_until isn't used (perhaps you've forgotten to + use "assert"). +* E7003: The argument given to "wait_until" must be a callable (such as + a function or lambda expression). +* E7004: FrameObject properties must always provide "self._frame" as the + "frame" parameter to functions such as "stbt.match". +* E7005: The image path given to "stbt.match" (and similar functions) + exists on disk, but isn't committed to git. +* E7006: FrameObject properties must use "self._frame", not + "stbt.get_frame()". +* E7007: FrameObject properties must not have side-effects that change + the state of the device-under-test by calling "stbt.press()" or + "stbt.press_and_wait()". +* E7008: "assert True" has no effect. + +""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import argparse +import re +import os +import subprocess +import sys +import threading + + +def main(argv): + parser = argparse.ArgumentParser( + prog="stbt lint", + usage="stbt lint [--help] [pylint options] filename [filename...]", + description=__doc__, + epilog="Any other command-line arguments are passed through to pylint.", + formatter_class=argparse.RawDescriptionHelpFormatter) + _, pylint_args = parser.parse_known_args(argv[1:]) + + if not pylint_args: + parser.print_usage(sys.stderr) + return 1 + + if sys.version_info.major == 2: + executable_name = "pylint" + else: + executable_name = "pylint3" + + try: + with open("/dev/null", "w") as devnull: + subprocess.check_call([executable_name, "--help"], + stdout=devnull, stderr=devnull) + except OSError as e: + if e.errno == 2: + sys.stderr.write( + "stbt lint: error: Couldn't find '%s' executable\n" + % executable_name) + return 1 + + pylint = subprocess.Popen( + [executable_name, "--load-plugins=_stbt.pylint_plugin"] + pylint_args, + stderr=subprocess.PIPE) + + t = threading.Thread(target=filter_warnings, + args=(pylint.stderr, + os.fdopen(sys.stderr.fileno(), "wb", 0))) + t.start() + + pylint.wait() + t.join() + return pylint.returncode + + +def filter_warnings(input_, output): + while True: + line = input_.readline() + if not line: + break + if any(re.search(pattern, line) for pattern in WARNINGS): + continue + output.write(line) + + +WARNINGS = [ + # pylint:disable=line-too-long + br"libdc1394 error: Failed to initialize libdc1394", + br"pygobject_register_sinkfunc is deprecated", + br"assertion .G_TYPE_IS_BOXED $boxed_type$. failed", + br"assertion .G_IS_PARAM_SPEC $pspec$. failed", + br"return isinstance$object, \(type, types.ClassType$\)", + br"return isinstance$object, type$", + br"gsignal.c:.*: parameter 1 of type '' for signal \".*\" is not a value type", + br"astroid.* Use gi.require_version", + br"^ __import__$m$$", +] + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-match stb-tester-31/stbt-match --- stb-tester-30-5-gbefe47c/stbt-match 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-match 1970-01-01 00:00:00.000000000 +0000 @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -""" -Copyright 2013 YouView TV Ltd. -License: LGPL v2.1 or (at your option) any later version (see -https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). -""" - -import argparse -import sys -from contextlib import contextmanager - -import cv2 - -import _stbt.logging -import stbt - - -def error(s): - sys.stderr.write("stbt match: error: %s\n" % s) - sys.exit(1) - - -parser = argparse.ArgumentParser() -parser.prog = "stbt match" -parser.description = """Run stbt's image-matching algorithm against a single - frame (which you can capture using `stbt screenshot`).""" -parser.add_argument( - "-v", "--verbose", action="store_true", - help="Dump image processing debug images to ./stbt-debug directory") -parser.add_argument( - "--all", action="store_true", - help='Use "stbt.match_all" instead of "stbt.match"') -parser.add_argument( - "source_file", help="""The screenshot to compare against (you can capture it - using 'stbt screenshot')""") -parser.add_argument( - "reference_file", help="The image to search for") -parser.add_argument( - "match_parameters", nargs="*", - help="""Parameters for the image processing algorithm. See - 'MatchParameters' in the stbt API documentation. For example: - 'confirm_threshold=0.70')""") -args = parser.parse_args(sys.argv[1:]) - -mp = {} -try: - for p in args.match_parameters: - name, value = p.split("=") - if name == "match_method": - mp["match_method"] = value - elif name == "match_threshold": - mp["match_threshold"] = float(value) - elif name == "confirm_method": - mp["confirm_method"] = value - elif name == "confirm_threshold": - mp["confirm_threshold"] = float(value) - elif name == "erode_passes": - mp["erode_passes"] = int(value) - else: - raise Exception("Unknown match_parameter argument '%s'" % p) -except Exception: # pylint:disable=broad-except - error("Invalid argument '%s'" % p) - -source_image = cv2.imread(args.source_file) -if source_image is None: - error("Invalid image '%s'" % args.source_file) - - -@contextmanager -def noop_contextmanager(): - yield - - -with (_stbt.logging.scoped_debug_level(2) if args.verbose - else noop_contextmanager()): - try: - match_found = False - for result in stbt.match_all( - args.reference_file, frame=source_image, - match_parameters=stbt.MatchParameters(**mp)): - print "%s: %s" % ( - "Match found" if result else "No match found. Closest match", - result) - if result: - match_found = True - if not args.all: - break - sys.exit(0 if match_found else 1) - except stbt.UITestError as e: - error(e.message) diff -Nru stb-tester-30-5-gbefe47c/stbt_match.py stb-tester-31/stbt_match.py --- stb-tester-30-5-gbefe47c/stbt_match.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_match.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,97 @@ +#!/usr/bin/python + +""" +Copyright 2013 YouView TV Ltd. +License: LGPL v2.1 or (at your option) any later version (see +https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). +""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import argparse +import sys +from contextlib import contextmanager + +import cv2 + +import _stbt.logging +import stbt + + +def error(s): + sys.stderr.write("stbt match: error: %s\n" % s) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.prog = "stbt match" + parser.description = """Run stbt's image-matching algorithm against a single + frame (which you can capture using `stbt screenshot`).""" + parser.add_argument( + "-v", "--verbose", action="store_true", + help="Dump image processing debug images to ./stbt-debug directory") + parser.add_argument( + "--all", action="store_true", + help='Use "stbt.match_all" instead of "stbt.match"') + parser.add_argument( + "source_file", help="""The screenshot to compare against (you can + capture it using 'stbt screenshot')""") + parser.add_argument( + "reference_file", help="The image to search for") + parser.add_argument( + "match_parameters", nargs="*", + help="""Parameters for the image processing algorithm. See + 'MatchParameters' in the stbt API documentation. For example: + 'confirm_threshold=0.70')""") + args = parser.parse_args(sys.argv[1:]) + + mp = {} + try: + for p in args.match_parameters: + name, value = p.split("=") + if name == "match_method": + mp["match_method"] = value + elif name == "match_threshold": + mp["match_threshold"] = float(value) + elif name == "confirm_method": + mp["confirm_method"] = value + elif name == "confirm_threshold": + mp["confirm_threshold"] = float(value) + elif name == "erode_passes": + mp["erode_passes"] = int(value) + else: + raise Exception("Unknown match_parameter argument '%s'" % p) + except Exception: # pylint:disable=broad-except + error("Invalid argument '%s'" % p) + + source_image = cv2.imread(args.source_file) + if source_image is None: + error("Invalid image '%s'" % args.source_file) + + with (_stbt.logging.scoped_debug_level(2) if args.verbose + else noop_contextmanager()): + match_found = False + for result in stbt.match_all( + args.reference_file, frame=source_image, + match_parameters=stbt.MatchParameters(**mp)): + print("%s: %s" % ( + "Match found" if result else "No match found. Closest match", + result)) + if result: + match_found = True + if not args.all: + break + sys.exit(0 if match_found else 1) + + +@contextmanager +def noop_contextmanager(): + yield + + +if __name__ == "__main__": + main() diff -Nru stb-tester-30-5-gbefe47c/stbt-power stb-tester-31/stbt-power --- stb-tester-30-5-gbefe47c/stbt-power 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-power 1970-01-01 00:00:00.000000000 +0000 @@ -1,57 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -from textwrap import dedent - -from _stbt.config import get_config -from _stbt.power import uri_to_power_outlet - - -def main(argv): - parser = argparse.ArgumentParser( - description="Control and query a computer controllable power outlet", - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--power-outlet", metavar="URI", - default=get_config("global", "power_outlet", ""), help=dedent("""\ - Address of the power device and the outlet on the device. - The format of is either: - aten:: - For ATEN network controlled PDU - ipp:: - For IP Power 9258 network controlled PDU - pdu:: - For PDUeX KWX network controlled PDU - rittal::: - For Rittal 7955.310 network controlled PDU - aviosys-8800-pro[:] - For Aviosys 8800 Pro USB - controlled outlets - where - The device's network address. - Address of the individual power outlet on - the device. Allowed values depend on the - specific device model. - The device name of the serial device that the - 8800 Pro exposes. Defaults to /dev/ttyACM0 - This URI defaults to from stbt.conf's "global.power_outlet" if not - specified on the command line. - """)) - - parser.add_argument( - "command", choices=["on", "off", "status"], metavar="command", - help=dedent("""\ - on|off: Turn power on or off - status: Prints ON if the outlet is powered, otherwise prints OFF - """)) - args = parser.parse_args(argv[1:]) - - outlet = uri_to_power_outlet(args.power_outlet) - - if args.command == "on": - outlet.set(True) - elif args.command == "off": - outlet.set(False) - elif args.command == "status": - sys.stdout.write("ON\n" if outlet.get() else "OFF\n") - else: - assert False - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt_power.py stb-tester-31/stbt_power.py --- stb-tester-30-5-gbefe47c/stbt_power.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_power.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,62 @@ +#!/usr/bin/python + +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order +import argparse +import sys +from textwrap import dedent + +from _stbt.config import get_config +from _stbt.power import uri_to_power_outlet + + +def main(argv): + parser = argparse.ArgumentParser( + description="Control and query a computer controllable power outlet", + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--power-outlet", metavar="URI", + default=get_config("global", "power_outlet", ""), help=dedent("""\ + Address of the power device and the outlet on the device. + The format of is either: + aten:: - For ATEN network controlled PDU + ipp:: - For IP Power 9258 network controlled PDU + pdu:: - For PDUeX KWX network controlled PDU + rittal::: - For Rittal 7955.310 network controlled PDU + aviosys-8800-pro[:] - For Aviosys 8800 Pro USB + controlled outlets + where + The device's network address. + Address of the individual power outlet on + the device. Allowed values depend on the + specific device model. + The device name of the serial device that the + 8800 Pro exposes. Defaults to /dev/ttyACM0 + This URI defaults to from stbt.conf's "global.power_outlet" if not + specified on the command line. + """)) + + parser.add_argument( + "command", choices=["on", "off", "status"], metavar="command", + help=dedent("""\ + on|off: Turn power on or off + status: Prints ON if the outlet is powered, otherwise prints OFF + """)) + args = parser.parse_args(argv[1:]) + + outlet = uri_to_power_outlet(args.power_outlet) + + if args.command == "on": + outlet.set(True) + elif args.command == "off": + outlet.set(False) + elif args.command == "status": + sys.stdout.write("ON\n" if outlet.get() else "OFF\n") + else: + assert False + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-record stb-tester-31/stbt-record --- stb-tester-30-5-gbefe47c/stbt-record 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-record 1970-01-01 00:00:00.000000000 +0000 @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -""" -Copyright 2012-2013 YouView TV Ltd. -License: LGPL v2.1 or (at your option) any later version (see -https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). -""" - -import itertools -import sys - -import _stbt.control -import _stbt.core -import stbt - - -def main(argv): - parser = _stbt.core.argparser() - parser.prog = 'stbt record' - parser.description = 'Create an stb-tester test script' - parser.add_argument( - '--control-recorder', - default=stbt.get_config('record', 'control_recorder'), - help='The source of remote control keypresses (default: %(default)s)') - parser.add_argument( - '-o', '--output-file', - default=stbt.get_config('record', 'output_file'), - help='The filename of the generated script (default: %(default)s)') - args = parser.parse_args(argv[1:]) - stbt.debug("Arguments:\n" + "\n".join([ - "%s: %s" % (k, v) for k, v in args.__dict__.items()])) - - try: - script = open(args.output_file, 'w') - except IOError as e: - e.strerror = "Failed to write to output-file '%s': %s" % ( - args.output_file, e.strerror) - raise - - with _stbt.core.new_device_under_test_from_config(args) as dut: - record(dut, args.control_recorder, script) - - -def record(dut, control_recorder, script_out): - dut.get_frame() # Fail early if no video - count = itertools.count(1) - old_key = None - - def write_wait_for_match(): - if old_key is None: - return - filename = "%04d-%s-complete.png" % (count.next(), old_key) - stbt.save_frame(dut.get_frame(), filename) - script_out.write(" stbt.wait_for_match('%s')\n" % filename) - - script_out.write("import stbt\n\n\n") - script_out.write("def test_that_WRITE_TESTCASE_DESCRIPTION_HERE():\n") - try: - for key in _stbt.control.uri_to_control_recorder(control_recorder): - write_wait_for_match() - script_out.write(" stbt.press('%s')\n" % key) - dut.press(key) - old_key = key - except KeyboardInterrupt: - write_wait_for_match() - return - write_wait_for_match() - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt_record.py stb-tester-31/stbt_record.py --- stb-tester-30-5-gbefe47c/stbt_record.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_record.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,76 @@ +#!/usr/bin/python + +""" +Copyright 2012-2013 YouView TV Ltd. +License: LGPL v2.1 or (at your option) any later version (see +https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). +""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import itertools +import sys + +import _stbt.control +import _stbt.core +import stbt + + +def main(argv): + parser = _stbt.core.argparser() + parser.prog = 'stbt record' + parser.description = 'Create an stb-tester test script' + parser.add_argument( + '--control-recorder', + default=stbt.get_config('record', 'control_recorder'), + help='The source of remote control keypresses (default: %(default)s)') + parser.add_argument( + '-o', '--output-file', + default=stbt.get_config('record', 'output_file'), + help='The filename of the generated script (default: %(default)s)') + args = parser.parse_args(argv[1:]) + stbt.debug("Arguments:\n" + "\n".join([ + "%s: %s" % (k, v) for k, v in args.__dict__.items()])) + + try: + script = open(args.output_file, 'w') + except IOError as e: + e.strerror = "Failed to write to output-file '%s': %s" % ( + args.output_file, e.strerror) + raise + + with _stbt.core.new_device_under_test_from_config(args) as dut: + record(dut, args.control_recorder, script) + + +def record(dut, control_recorder, script_out): + dut.get_frame() # Fail early if no video + count = itertools.count(1) + old_key = None + + def write_wait_for_match(): + if old_key is None: + return + filename = "%04d-%s-complete.png" % (next(count), old_key) + stbt.save_frame(dut.get_frame(), filename) + script_out.write(" stbt.wait_for_match('%s')\n" % filename) + + script_out.write("import stbt\n\n\n") + script_out.write("def test_that_WRITE_TESTCASE_DESCRIPTION_HERE():\n") + try: + for key in _stbt.control.uri_to_control_recorder(control_recorder): + write_wait_for_match() + script_out.write(" stbt.press('%s')\n" % key) + dut.press(key) + old_key = key + except KeyboardInterrupt: + write_wait_for_match() + return + write_wait_for_match() + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-run stb-tester-31/stbt-run --- stb-tester-30-5-gbefe47c/stbt-run 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-run 1970-01-01 00:00:00.000000000 +0000 @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -""" -Copyright 2012-2013 YouView TV Ltd. - 2014-2017 stb-tester.com Ltd. -License: LGPL v2.1 or (at your option) any later version (see -https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). -""" - -import argparse -import sys - -import _stbt.core -import stbt -from _stbt.stbt_run import (load_test_function, - sane_unicode_and_exception_handling, video) - - -def main(argv): - parser = _stbt.core.argparser() - parser.prog = 'stbt run' - parser.description = 'Run an stb-tester test script' - parser.add_argument( - '--save-screenshot', default='on-failure', - choices=['always', 'on-failure', 'never'], - help="Save a screenshot at the end of the test to screenshot.png") - parser.add_argument( - '--save-thumbnail', default='never', - choices=['always', 'on-failure', 'never'], - help="Save a thumbnail at the end of the test to thumbnail.jpg") - parser.add_argument( - 'script', metavar='FILE[::TESTCASE]', help=( - "The python test script to run. Optionally specify a python " - "function name to run that function; otherwise only the script's " - "top-level will be executed.")) - parser.add_argument( - 'args', nargs=argparse.REMAINDER, metavar='ARG', - help='Additional arguments passed on to the test script (in sys.argv)') - - args = parser.parse_args(argv[1:]) - stbt.debug("Arguments:\n" + "\n".join([ - "%s: %s" % (k, v) for k, v in args.__dict__.items()])) - - dut = _stbt.core.new_device_under_test_from_config(args) - with sane_unicode_and_exception_handling(args.script), video(args, dut): - test_function = load_test_function(args.script, args.args) - test_function.call() - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt_run.py stb-tester-31/stbt_run.py --- stb-tester-30-5-gbefe47c/stbt_run.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/stbt_run.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,56 @@ +#!/usr/bin/python + +""" +Copyright 2012-2013 YouView TV Ltd. + 2014-2017 stb-tester.com Ltd. +License: LGPL v2.1 or (at your option) any later version (see +https://github.com/stb-tester/stb-tester/blob/master/LICENSE for details). +""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import argparse +import sys + +import _stbt.core +import stbt +from _stbt.stbt_run import (load_test_function, + sane_unicode_and_exception_handling, video) + + +def main(argv): + parser = _stbt.core.argparser() + parser.prog = 'stbt run' + parser.description = 'Run an stb-tester test script' + parser.add_argument( + '--save-screenshot', default='on-failure', + choices=['always', 'on-failure', 'never'], + help="Save a screenshot at the end of the test to screenshot.png") + parser.add_argument( + '--save-thumbnail', default='never', + choices=['always', 'on-failure', 'never'], + help="Save a thumbnail at the end of the test to thumbnail.jpg") + parser.add_argument( + 'script', metavar='FILE[::TESTCASE]', help=( + "The python test script to run. Optionally specify a python " + "function name to run that function; otherwise only the script's " + "top-level will be executed.")) + parser.add_argument( + 'args', nargs=argparse.REMAINDER, metavar='ARG', + help='Additional arguments passed on to the test script (in sys.argv)') + + args = parser.parse_args(argv[1:]) + stbt.debug("Arguments:\n" + "\n".join([ + "%s: %s" % (k, v) for k, v in args.__dict__.items()])) + + dut = _stbt.core.new_device_under_test_from_config(args) + with sane_unicode_and_exception_handling(args.script), video(args, dut): + test_function = load_test_function(args.script, args.args) + test_function.call() + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff -Nru stb-tester-30-5-gbefe47c/stbt-screenshot stb-tester-31/stbt-screenshot --- stb-tester-30-5-gbefe47c/stbt-screenshot 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-screenshot 2019-09-18 14:04:32.000000000 +0000 @@ -17,12 +17,12 @@ filename=${1:-screenshot.png} -"$(dirname "$0")/stbt-run" -v --control=none --save-video= <(cat <<-EOF +"$(dirname "$0")/stbt_run.py" -v --control=none --save-video= <(cat <<-EOF import time get_frame() if stbt._dut._display.source_pipeline_description.startswith("decklink"): time.sleep(2) # First second of Blackmagic video has purple tint save_frame(get_frame(), "$filename") - print "Screenshot saved to '$filename'" + print("Screenshot saved to '$filename'") EOF ) diff -Nru stb-tester-30-5-gbefe47c/stbt-tv stb-tester-31/stbt-tv --- stb-tester-30-5-gbefe47c/stbt-tv 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt-tv 2019-09-18 14:04:32.000000000 +0000 @@ -53,10 +53,9 @@ shift done -get_config () { "$script_dir/stbt-config" "$1" ; } +get_config () { "$script_dir/stbt_config.py" "$1" ; } source_pipeline="${*:-$(get_config global.source_pipeline)}" -transformation_pipeline="$(get_config global.transformation_pipeline)" : ${sink:=$( case "$(uname)" in @@ -76,6 +75,5 @@ ! queue name=_stbt_raw_frames_queue max-size-buffers=2 leaky=upstream \ ! videoconvert \ ! video/x-raw,format=BGR \ - ! $transformation_pipeline \ ! videoconvert \ ! $sink sync=false diff -Nru stb-tester-30-5-gbefe47c/stbt_virtual_stb.py stb-tester-31/stbt_virtual_stb.py --- stb-tester-30-5-gbefe47c/stbt_virtual_stb.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/stbt_virtual_stb.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python """ stbt virtual-stb enables stb-tester to test set-top box software without hardware. This can be useful as a first stage in a continuous integration @@ -24,6 +24,11 @@ # Tear down the chromium virtual-stb and remove the configuration: stbt virtual-stb stop """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import argparse import errno diff -Nru stb-tester-30-5-gbefe47c/tests/Apple_TV.lircd.conf stb-tester-31/tests/Apple_TV.lircd.conf --- stb-tester-30-5-gbefe47c/tests/Apple_TV.lircd.conf 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/tests/Apple_TV.lircd.conf 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,22 @@ +begin remote + name Apple_TV + bits 32 + flags SPACE_ENC + frequency 38000 + header 9000 4500 + one 527 1703 + zero 527 527 + ptrail 527 + gap 38000 + + begin codes + KEY_DOWN 0x77E1306E + KEY_LEFT 0x77E1906E + KEY_BACK 0x77E1C06E + KEY_OK 0x77E13A6E + KEY_PLAY 0x77E1FA6E + KEY_RIGHT 0x77E1606E + KEY_UP 0x77E1506E + end codes + +end remote diff -Nru stb-tester-30-5-gbefe47c/tests/auto_selftest_bare.py stb-tester-31/tests/auto_selftest_bare.py --- stb-tester-30-5-gbefe47c/tests/auto_selftest_bare.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto_selftest_bare.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -# coding=utf-8 - - -def hello(): - r""" - Basics: - - >>> True - >>> False - >>> None - >>> print - >>> print "Hello" - >>> a = "hello" - >>> raise Exception("bye-bye") - - Unicode: - - >>> print "Spın̈al Tap" - >>> print u"Spın̈al Tap" - >>> print u"Spinal Tap" - >>> "Spın̈al Tap" - >>> u"Spın̈al Tap" - >>> raise Exception("Spın̈al Tap") - >>> raise Exception(u"Spın̈al Tap") - - Removal of uninteresting tests: - - >>> True # remove-if-false - >>> False # remove-if-false - >>> raise Exception("FORE! I mean FIVE! I mean FIRE!") # remove-if-false - >>> print "hello" # remove-if-false - >>> a = "hello" # remove-if-false - """ - pass diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/README stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/README --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/README 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -This directory contains self-tests generated by `stbt auto-selftest`. -Do not modify by hand. Any files modified or created in this directory -may be overwritten or deleted by `stbt auto-selftest`. diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/example_selftest.py stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/example_selftest.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/example_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/example_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -This file contains regression tests automatically generated by -``stbt auto-selftest``. These tests are intended to capture the -behaviour of Frame Objects (and other helper functions that operate on -a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` -whenever you make a change to your Frame Objects, and use ``git diff`` -to see how your changes affect the behaviour of the Frame Object. - -NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- -it merely documents the behaviour at the time that -``stbt auto-selftest`` was run. -""" -# pylint: disable=line-too-long - -import os -import sys - -sys.path.insert(0, os.path.join( - os.path.dirname(__file__), '../../../tests')) - -from example import * # isort:skip pylint: disable=wildcard-import, import-error - -_FRAME_CACHE = {} - - -def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - '../../screenshots', name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - - -def auto_selftest_Dialog(): - r""" - >>> Dialog(frame=f("frame-object-with-dialog-different-background.png")) - Dialog(is_visible=True, message=u'This set-top box is great') - >>> Dialog(frame=f("frame-object-with-dialog.png")) - Dialog(is_visible=True, message=u'This set-top box is great') - >>> Dialog(frame=f("frame-object-with-dialog2.png")) - Dialog(is_visible=True, message=u'This set-top box is fabulous') - """ - pass - - -def auto_selftest_FalseyFrameObject(): - r""" - >>> FalseyFrameObject(frame=f("frame-object-with-dialog-different-background.png")) - FalseyFrameObject(is_visible=False) - >>> FalseyFrameObject(frame=f("frame-object-with-dialog.png")) - FalseyFrameObject(is_visible=False) - >>> FalseyFrameObject(frame=f("frame-object-with-dialog2.png")) - FalseyFrameObject(is_visible=False) - """ - pass - - -def auto_selftest_TruthyFrameObject2(): - r""" - >>> TruthyFrameObject2(frame=f("frame-object-without-dialog-different-background.png")) - TruthyFrameObject2(is_visible=True) - >>> TruthyFrameObject2(frame=f("frame-object-without-dialog.png")) - TruthyFrameObject2(is_visible=True) - """ - pass - - -def auto_selftest_not_a_frame_object(): - r""" - >>> not_a_frame_object(4, f("frame-object-with-dialog-different-background.png")) - hello 4 - True - >>> not_a_frame_object(4, f("frame-object-with-dialog.png")) - hello 4 - True - >>> not_a_frame_object(4, f("frame-object-with-dialog2.png")) - hello 4 - True - >>> not_a_frame_object(4, f("frame-object-without-dialog-different-background.png")) - hello 4 - True - >>> not_a_frame_object(4, f("frame-object-without-dialog.png")) - hello 4 - True - >>> not_a_frame_object(2, f("frame-object-with-dialog-different-background.png")) - hello 2 - True - >>> not_a_frame_object(2, f("frame-object-with-dialog.png")) - hello 2 - True - >>> not_a_frame_object(2, f("frame-object-with-dialog2.png")) - hello 2 - True - >>> not_a_frame_object(2, f("frame-object-without-dialog-different-background.png")) - hello 2 - True - >>> not_a_frame_object(2, f("frame-object-without-dialog.png")) - hello 2 - True - """ - pass diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/package/a_selftest.py stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/package/a_selftest.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/package/a_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/package/a_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -This file contains regression tests automatically generated by -``stbt auto-selftest``. These tests are intended to capture the -behaviour of Frame Objects (and other helper functions that operate on -a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` -whenever you make a change to your Frame Objects, and use ``git diff`` -to see how your changes affect the behaviour of the Frame Object. - -NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- -it merely documents the behaviour at the time that -``stbt auto-selftest`` was run. -""" -# pylint: disable=line-too-long - -import os -import sys - -sys.path.insert(0, os.path.join( - os.path.dirname(__file__), '../../../../tests')) - -from package.a import * # isort:skip pylint: disable=wildcard-import, import-error - -_FRAME_CACHE = {} - - -def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - '../../../screenshots', name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - - -def auto_selftest_GreetingFrameObject(): - r""" - >>> GreetingFrameObject(frame=f("frame-object-with-dialog-different-background.png")) - GreetingFrameObject(is_visible=True, greeting='hi!') - >>> GreetingFrameObject(frame=f("frame-object-with-dialog.png")) - GreetingFrameObject(is_visible=True, greeting='hi!') - >>> GreetingFrameObject(frame=f("frame-object-with-dialog2.png")) - GreetingFrameObject(is_visible=True, greeting='hi!') - >>> GreetingFrameObject(frame=f("frame-object-without-dialog-different-background.png")) - GreetingFrameObject(is_visible=True, greeting='hi!') - >>> GreetingFrameObject(frame=f("frame-object-without-dialog.png")) - GreetingFrameObject(is_visible=True, greeting='hi!') - """ - pass diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/subdir/subsubdir/subdir_example_selftest.py stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/subdir/subsubdir/subdir_example_selftest.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/subdir/subsubdir/subdir_example_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/subdir/subsubdir/subdir_example_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -This file contains regression tests automatically generated by -``stbt auto-selftest``. These tests are intended to capture the -behaviour of Frame Objects (and other helper functions that operate on -a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` -whenever you make a change to your Frame Objects, and use ``git diff`` -to see how your changes affect the behaviour of the Frame Object. - -NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- -it merely documents the behaviour at the time that -``stbt auto-selftest`` was run. -""" -# pylint: disable=line-too-long - -import os -import sys - -sys.path.insert(0, os.path.join( - os.path.dirname(__file__), '../../../../../tests/subdir/subsubdir')) - -from subdir_example import * # isort:skip pylint: disable=wildcard-import, import-error - -_FRAME_CACHE = {} - - -def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - '../../../../screenshots', name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - - -def auto_selftest_Truth(): - r""" - >>> Truth(frame=f("frame-object-with-dialog-different-background.png")) - Truth(is_visible=True) - >>> Truth(frame=f("frame-object-with-dialog.png")) - Truth(is_visible=True) - >>> Truth(frame=f("frame-object-with-dialog2.png")) - Truth(is_visible=True) - >>> Truth(frame=f("frame-object-without-dialog-different-background.png")) - Truth(is_visible=True) - >>> Truth(frame=f("frame-object-without-dialog.png")) - Truth(is_visible=True) - """ - pass diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/unicode_example_selftest.py stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/unicode_example_selftest.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/unicode_example_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests/unicode_example_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -This file contains regression tests automatically generated by -``stbt auto-selftest``. These tests are intended to capture the -behaviour of Frame Objects (and other helper functions that operate on -a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` -whenever you make a change to your Frame Objects, and use ``git diff`` -to see how your changes affect the behaviour of the Frame Object. - -NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- -it merely documents the behaviour at the time that -``stbt auto-selftest`` was run. -""" -# pylint: disable=line-too-long - -import os -import sys - -sys.path.insert(0, os.path.join( - os.path.dirname(__file__), '../../../tests')) - -from unicode_example import * # isort:skip pylint: disable=wildcard-import, import-error - -_FRAME_CACHE = {} - - -def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - '../../screenshots', name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - - -def auto_selftest_identity(): - r""" - >>> "Spinal Tap" - 'Spinal Tap' - >>> u"Spinal Tap" - u'Spinal Tap' - >>> "Spın̈al Tap" - 'Sp\xc4\xb1n\xcc\x88al Tap' - >>> u"Spın̈al Tap" - u'Sp\xc4\xb1n\xcc\x88al Tap' - >>> print "Spinal Tap" - Spinal Tap - >>> print u"Spinal Tap" - Spinal Tap - >>> print "Spın̈al Tap" - Spın̈al Tap - >>> print u"Spın̈al Tap" # doctest: +SKIP - Sp\xc4\xb1n\xcc\x88al Tap - >>> raise Exception("Spinal Tap") - Traceback (most recent call last): - Exception: Spinal Tap - >>> raise Exception(u"Spinal Tap") - Traceback (most recent call last): - Exception: Spinal Tap - >>> raise Exception("Spın̈al Tap") - Traceback (most recent call last): - Exception: Spın̈al Tap - >>> raise Exception(u"Spın̈al Tap") - Traceback (most recent call last): - Exception: Sp\xc4\xb1n\xcc\x88al Tap - """ - pass - - -def auto_selftest_print_something_unicode(): - r""" - >>> print_something_unicode(0) - Spinal Tap - >>> print_something_unicode(1) - Spinal Tap - >>> print_something_unicode(2) - Spın̈al Tap - >>> print_something_unicode(3) # doctest: +SKIP - Sp\u0131n\u0308al Tap - """ - pass - - -def auto_selftest_return_something_unicode(): - r""" - >>> return_something_unicode(0) - 'Spinal Tap' - >>> return_something_unicode(1) - u'Spinal Tap' - >>> return_something_unicode(2) - 'Sp\xc4\xb1n\xcc\x88al Tap' - >>> return_something_unicode(3) - u'Sp\u0131n\u0308al Tap' - >>> print return_something_unicode(0) - Spinal Tap - >>> print return_something_unicode(1) - Spinal Tap - >>> print return_something_unicode(2) - Spın̈al Tap - >>> print return_something_unicode(3) # doctest: +SKIP - Sp\u0131n\u0308al Tap - """ - pass diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests_in_root_selftest.py stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests_in_root_selftest.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests_in_root_selftest.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/selftest/auto_selftest/tests_in_root_selftest.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -This file contains regression tests automatically generated by -``stbt auto-selftest``. These tests are intended to capture the -behaviour of Frame Objects (and other helper functions that operate on -a video-frame). Commit this file to git, re-run ``stbt auto-selftest`` -whenever you make a change to your Frame Objects, and use ``git diff`` -to see how your changes affect the behaviour of the Frame Object. - -NOTE: THE OUTPUT OF THE DOCTESTS BELOW IS NOT NECESSARILY "CORRECT" -- -it merely documents the behaviour at the time that -``stbt auto-selftest`` was run. -""" -# pylint: disable=line-too-long - -import os -import sys - -sys.path.insert(0, os.path.join( - os.path.dirname(__file__), '../..')) - -from tests_in_root import * # isort:skip pylint: disable=wildcard-import, import-error - -_FRAME_CACHE = {} - - -def f(name): - img = _FRAME_CACHE.get(name) - if img is None: - import cv2 - filename = os.path.join(os.path.dirname(__file__), - '../screenshots', name) - img = cv2.imread(filename) - assert img is not None, "Failed to load %s" % filename - img.flags.writeable = False - _FRAME_CACHE[name] = img - return img - - -def auto_selftest_Dialog(): - r""" - >>> Dialog(frame=f("frame-object-with-dialog-different-background.png")) - Dialog(is_visible=True, message=u'This set-top box is great') - >>> Dialog(frame=f("frame-object-with-dialog.png")) - Dialog(is_visible=True, message=u'This set-top box is great') - >>> Dialog(frame=f("frame-object-with-dialog2.png")) - Dialog(is_visible=True, message=u'This set-top box is fabulous') - """ - pass Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog2.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog2.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog-different-background.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog-different-background.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-with-dialog.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-without-dialog-different-background.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-without-dialog-different-background.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-without-dialog.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/selftest/screenshots/frame-object-without-dialog.png differ diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/.stbt.conf stb-tester-31/tests/auto-selftest-example-test-pack/.stbt.conf --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/.stbt.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/.stbt.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -[match] -# Disable performance optimisation to make the effect of `stbt auto-selftest` -# caching more obvious. -pyramid_levels = 1 diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/bad-module-name.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/bad-module-name.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/bad-module-name.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/bad-module-name.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,7 +0,0 @@ -import stbt - - -class Truth(stbt.FrameObject): - @property - def is_visible(self): - return True diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/empty_dir/subdir/example_with_no_tests.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/empty_dir/subdir/example_with_no_tests.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/empty_dir/subdir/example_with_no_tests.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/empty_dir/subdir/example_with_no_tests.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -# There should be no selftest file generated for this file because there are -# no tests to be generated, but also the directory it's in should be deleted. diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/example.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/example.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/example.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/example.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,64 +0,0 @@ -import stbt - - -# This is a basic frame object that will be run against all our example -# screenshots. stbt auto-selftest will generate doctests for each of the -# screenshots it matches and not for the screenshots it doesn't. -class Dialog(stbt.FrameObject): - @property - def is_visible(self): - return bool(stbt.match('info.png', frame=self._frame)) - - @property - def message(self): - return stbt.ocr(region=stbt.Region(515, 331, 400, 100), - frame=self._frame).replace('\n', ' ') - - -# Sometimes we want to force doctests to be written out even if they are falsey. -# This is particularly true if we've added a screenshot that illustrates an -# example that our FrameObject should match but doesn't. To enforce this we -# set AUTO_SELFTEST_SCREENSHOTS. -class FalseyFrameObject(stbt.FrameObject): - AUTO_SELFTEST_SCREENSHOTS = ['*-with-*.png'] - - @property - def is_visible(self): - return False - - -# If we want an item in a module to be tested we just need to add a -# AUTO_SELFTEST_EXPRESSIONS member to it. In fact this is exactly why -# stbt auto-selftest knows to test FrameObjects: the FrameObject base class -# defines this member for you. -# -# Here's an example of testing a function instead of a class: -def not_a_frame_object(name, _): - print "hello %s" % name - return True - -not_a_frame_object.AUTO_SELFTEST_EXPRESSIONS = [ - 'not_a_frame_object(4, {frame})', - 'not_a_frame_object(2, {frame})', -] - - -# And to further illustrate the point here's an example of disabling -# auto-selftest for a FrameObject: -class TruthyFrameObject1(stbt.FrameObject): - AUTO_SELFTEST_EXPRESSIONS = [] - - @property - def is_visible(self): - return True - - -# By default we will try running the Frame Object against every screenshot in -# selftest/screenshots. We can explicitly set AUTO_SELFTEST_TRY_SCREENSHOTS -# to restrict the screenshots it's checked against. This defaults to ['*.png']: -class TruthyFrameObject2(stbt.FrameObject): - AUTO_SELFTEST_TRY_SCREENSHOTS = ['*-without-*.png'] - - @property - def is_visible(self): - return True diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/example_with_no_tests.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/example_with_no_tests.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/example_with_no_tests.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/example_with_no_tests.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ -# There should be no selftest file generated for this file because there are -# no tests to be generated. - -import stbt - - -class NoTestFrameObject(stbt.FrameObject): - AUTO_SELFTEST_EXPRESSIONS = [] - - @property - def is_visible(self): - return True - - -class NoScreenshotFrameObject(stbt.FrameObject): - AUTO_SELFTEST_TRY_SCREENSHOTS = [] - - @property - def is_visible(self): - return True diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/import_example.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/import_example.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/import_example.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/import_example.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# We don't want to be generating a test for Dialog as it is defined in a -# separate file where it will already have tests generated for it. -from example import Dialog # pylint: disable=unused-import Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/info.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/auto-selftest-example-test-pack/tests/info.png differ diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/package/a.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/package/a.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/package/a.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/package/a.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,13 +0,0 @@ -import stbt - -from .b import greeting - - -class GreetingFrameObject(stbt.FrameObject): - @property - def is_visible(self): - return True - - @property - def greeting(self): - return greeting() diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/package/b.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/package/b.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/package/b.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/package/b.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -def greeting(): - return "hi!" diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/subdir/subsubdir/subdir_example.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/subdir/subsubdir/subdir_example.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/subdir/subsubdir/subdir_example.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/subdir/subsubdir/subdir_example.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,7 +0,0 @@ -import stbt - - -class Truth(stbt.FrameObject): - @property - def is_visible(self): - return True diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/syntax_error.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/syntax_error.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/syntax_error.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/syntax_error.py 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -This file is invalid Python. stbt auto-selftest should ignore it. diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/unicode_example.py stb-tester-31/tests/auto-selftest-example-test-pack/tests/unicode_example.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests/unicode_example.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests/unicode_example.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,62 +0,0 @@ -# coding=utf-8 -# These are to test the unicode output of `stbt auto-selftest generate`: - - -def identity(x): - return x -identity.AUTO_SELFTEST_TRY_SCREENSHOTS = [] -identity.AUTO_SELFTEST_SCREENSHOTS = ["frame-object-with-dialog.png"] -identity.AUTO_SELFTEST_EXPRESSIONS = [ - '"Spinal Tap"', - 'u"Spinal Tap"', - '"Spın̈al Tap"', - 'u"Spın̈al Tap"', - 'print "Spinal Tap"', - 'print u"Spinal Tap"', - 'print "Spın̈al Tap"', - 'print u"Spın̈al Tap"', - 'raise Exception("Spinal Tap")', - 'raise Exception(u"Spinal Tap")', - 'raise Exception("Spın̈al Tap")', - 'raise Exception(u"Spın̈al Tap")', -] - - -def print_something_unicode(n): - print([ - "Spinal Tap", - u"Spinal Tap", - "Spın̈al Tap", - u"Spın̈al Tap", - ][n]) -print_something_unicode.AUTO_SELFTEST_TRY_SCREENSHOTS = [] -print_something_unicode.AUTO_SELFTEST_SCREENSHOTS = [ - "frame-object-with-dialog.png"] -print_something_unicode.AUTO_SELFTEST_EXPRESSIONS = [ - 'print_something_unicode(0)', - 'print_something_unicode(1)', - 'print_something_unicode(2)', - 'print_something_unicode(3)', -] - - -def return_something_unicode(n): - return [ - "Spinal Tap", - u"Spinal Tap", - "Spın̈al Tap", - u"Spın̈al Tap", - ][n] -return_something_unicode.AUTO_SELFTEST_TRY_SCREENSHOTS = [] -return_something_unicode.AUTO_SELFTEST_SCREENSHOTS = [ - "frame-object-with-dialog.png"] -return_something_unicode.AUTO_SELFTEST_EXPRESSIONS = [ - 'return_something_unicode(0)', - 'return_something_unicode(1)', - 'return_something_unicode(2)', - 'return_something_unicode(3)', - 'print return_something_unicode(0)', - 'print return_something_unicode(1)', - 'print return_something_unicode(2)', - 'print return_something_unicode(3)', -] diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests_in_root.py stb-tester-31/tests/auto-selftest-example-test-pack/tests_in_root.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tests_in_root.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tests_in_root.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,15 +0,0 @@ -import stbt - - -# This is a basic frame object that will be run against all our example -# screenshots. stbt auto-selftest will generate doctests for each of the -# screenshots it matches and not for the screenshots it doesn't. -class Dialog(stbt.FrameObject): - @property - def is_visible(self): - return bool(stbt.match('tests/info.png', frame=self._frame)) - - @property - def message(self): - return stbt.ocr(region=stbt.Region(515, 331, 400, 100), - frame=self._frame).replace('\n', ' ') diff -Nru stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tool.py stb-tester-31/tests/auto-selftest-example-test-pack/tool.py --- stb-tester-30-5-gbefe47c/tests/auto-selftest-example-test-pack/tool.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto-selftest-example-test-pack/tool.py 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -#!/usr/bin/env python diff -Nru stb-tester-30-5-gbefe47c/tests/auto_selftest_expected.py stb-tester-31/tests/auto_selftest_expected.py --- stb-tester-30-5-gbefe47c/tests/auto_selftest_expected.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/auto_selftest_expected.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,51 +0,0 @@ -# coding=utf-8 - - -def hello(): - r""" - Basics: - - >>> True - True - >>> False - False - >>> None - >>> print - - >>> print "Hello" - Hello - >>> a = "hello" - >>> raise Exception("bye-bye") - Traceback (most recent call last): - Exception: bye-bye - - Unicode: - - >>> print "Spın̈al Tap" - Spın̈al Tap - >>> print u"Spın̈al Tap" # doctest: +SKIP - Sp\xc4\xb1n\xcc\x88al Tap - >>> print u"Spinal Tap" - Spinal Tap - >>> "Spın̈al Tap" - 'Sp\xc4\xb1n\xcc\x88al Tap' - >>> u"Spın̈al Tap" - u'Sp\xc4\xb1n\xcc\x88al Tap' - >>> raise Exception("Spın̈al Tap") - Traceback (most recent call last): - Exception: Spın̈al Tap - >>> raise Exception(u"Spın̈al Tap") - Traceback (most recent call last): - Exception: Sp\xc4\xb1n\xcc\x88al Tap - - Removal of uninteresting tests: - - >>> True - True - >>> raise Exception("FORE! I mean FIVE! I mean FIRE!") - Traceback (most recent call last): - Exception: FORE! I mean FIVE! I mean FIRE! - >>> print "hello" - hello - """ - pass Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/custom-image-label.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/custom-image-label.png differ diff -Nru stb-tester-30-5-gbefe47c/tests/fake-irnetbox stb-tester-31/tests/fake-irnetbox --- stb-tester-30-5-gbefe47c/tests/fake-irnetbox 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/fake-irnetbox 2019-09-18 14:04:32.000000000 +0000 @@ -1,12 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/python """Fake irNetBox server for unit tests. Just ACKs all requests. See ../stbt/irnetbox.py for information on the irNetBox. """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order +import binascii import socket -import SocketServer +import socketserver import struct import sys import time @@ -14,21 +20,21 @@ from _stbt.irnetbox import MessageTypes -class FakeIRNetBox(SocketServer.BaseRequestHandler): +class FakeIRNetBox(socketserver.BaseRequestHandler): def handle(self): x = None - while x != "#": + while x != b"#": try: x = self.recv(1) except socket.error as e: if e.errno == socket.errno.ECONNRESET: - print "fake-irnetbox: Client closed connection" + print("fake-irnetbox: Client closed connection") return raise _ = struct.unpack(">H", self.recv(2)) message_type = ord(self.recv(1)) - print "fake-irnetbox: Received message %s (0x%02x)" % ( - name(message_type), message_type) + print("fake-irnetbox: Received message %s (0x%02x)" % ( + name(message_type), message_type)) if message_type in [ MessageTypes.POWER_ON, @@ -45,7 +51,7 @@ if message_type == MessageTypes.OUTPUT_IR_ASYNC: sequence_number, = struct.unpack(">H", self.recv(2)) - print "fake-irnetbox: ...with sequence number %d" % sequence_number + print("fake-irnetbox: ...with sequence number %d" % sequence_number) if self.server.response == "error": self.send( MessageTypes.ERROR, @@ -64,7 +70,7 @@ "HB%ds" % len(data), len(data), message_type, data) self.request.sendall(m) - print "fake-irnetbox: Sent response %s: %s" % ( - name(message_type), hexstring(m)) - - -def hexstring(s): - return " ".join(["%02x" % ord(x) for x in s]) + print("fake-irnetbox: Sent response %s: %s" % ( + name(message_type), binascii.hexlify(m))) def name(message_type): @@ -111,11 +113,11 @@ }.get(message_type, message_type) -class Server(SocketServer.ThreadingTCPServer): +class Server(socketserver.ThreadingTCPServer): daemon_threads = True def __init__(self, address, handler, response): - SocketServer.ThreadingTCPServer.__init__(self, address, handler) + socketserver.ThreadingTCPServer.__init__(self, address, handler) self.response = response @@ -124,5 +126,5 @@ ("localhost", 0), FakeIRNetBox, sys.argv[1] if len(sys.argv) > 1 else "ack") - print "PORT=%s" % server.socket.getsockname()[1] + print("PORT=%s" % server.socket.getsockname()[1]) server.serve_forever() diff -Nru stb-tester-30-5-gbefe47c/tests/fake-lircd stb-tester-31/tests/fake-lircd --- stb-tester-30-5-gbefe47c/tests/fake-lircd 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/fake-lircd 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python """Fake LIRC daemon for unit tests. @@ -11,6 +11,11 @@ See http://www.lirc.org/html/technical.html#applications """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import re @@ -25,17 +30,16 @@ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.bind(f) s.listen(5) - print "SOCKET=" + f + print("SOCKET=" + f) while True: client, _ = s.accept() - for line in client.makefile(): + for line in client.makefile("rb"): line = line.strip() - print "fake-lircd: Received: " + line + print("fake-lircd: Received: %s" % line) try: r = response(line) if r: - print "\n".join( - "fake-lircd: Sending: " + x for x in r.split("\n")) + print("fake-lircd: Sending: %s" % r) client.sendall(r) except socket.error as e: if e.errno == socket.errno.EPIPE: @@ -45,18 +49,18 @@ def response(req): - s = "" - if re.search("sighup", req): - s += "BEGIN\nSIGHUP\nEND\n" - if re.search("broadcast", req): - s += "000f00b 01 OK My-IR-remote\n" - if req.startswith("SEND_"): - if re.search("error", req): - s += "BEGIN\n%s\nERROR\nDATA\n1\nfake-lircd error\nEND\n" % req - elif re.search("timeout", req): + s = b"" + if re.search(b"sighup", req): + s += b"BEGIN\nSIGHUP\nEND\n" + if re.search(b"broadcast", req): + s += b"000f00b 01 OK My-IR-remote\n" + if req.startswith(b"SEND_"): + if re.search(b"error", req): + s += b"BEGIN\n%s\nERROR\nDATA\n1\nfake-lircd error\nEND\n" % req + elif re.search(b"timeout", req): pass else: - s += "BEGIN\n%s\nSUCCESS\nEND\n" % req + s += b"BEGIN\n%s\nSUCCESS\nEND\n" % req return s diff -Nru stb-tester-30-5-gbefe47c/tests/fake-video-src.py stb-tester-31/tests/fake-video-src.py --- stb-tester-30-5-gbefe47c/tests/fake-video-src.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/fake-video-src.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,93 +0,0 @@ -#!/usr/bin/python -u - -import argparse -import os -import sys -import threading - -import gi - -from _stbt import gst_utils, utils - -gi.require_version("Gst", "1.0") -from gi.repository import Gst # pylint:disable=wrong-import-order - -Gst.init([]) - -USE_SHMSRC = True - - -def main(argv): - parser = argparse.ArgumentParser() - parser.add_argument("socket", help="shmsrc socket") - - args = parser.parse_args(argv[1:]) - - cache_root = (os.environ.get("XDG_CACHE_HOME", None) or - os.environ.get("HOME") + '/.cache') - default_file = '%s/stbt/camera-video-cache/black.mp4' % cache_root - - if not os.path.exists(default_file): - utils.mkdir_p(os.path.dirname(default_file)) - gst_utils.frames_to_video( - default_file, [(bytearray([0, 0, 0]) * 1280 * 720, 5 * Gst.SECOND)], - 'video/x-raw,format=BGR,width=1280,height=720', 'mp4') - - default_uri = "file://" + default_file - - frame_bytes = 1280 * 720 * 3 - - next_video = [default_uri] - - def about_to_finish(playbin): - playbin.set_property('uri', next_video[0]) - next_video[0] = default_uri - playbin.set_state(Gst.State.PLAYING) - - if USE_SHMSRC: - pipeline_desc = ( - """\ - playbin name=pb audio-sink=fakesink uri=%s flags=0x00000791 \ - video-sink="videoconvert \ - ! video/x-raw,width=1280,height=720,format=RGB ! identity ! \ - shmsink wait-for-connection=true shm-size=%i max-lateness=-1 \ - qos=false socket-path=%s blocksize=%i sync=true \ - buffer-time=100000000" """ - % (default_uri, frame_bytes * 1000, args.socket, frame_bytes)) - else: - pipeline_desc = ( - """playbin name=pb audio-sink=fakesink uri=%s flags=0x00000791 \ - video-sink="videoconvert ! timeoverlay ! xvimagesink sync=true" """ - % default_uri) - - playbin = Gst.parse_launch(pipeline_desc) - - playbin.connect("about-to-finish", about_to_finish) - - runner = gst_utils.PipelineRunner(playbin) - gst_thread = threading.Thread(target=runner.run) - gst_thread.daemon = True - gst_thread.start() - - playbin.get_state(0) - - def set_uri(uri): - print "=== Setting URI to", uri - if uri == 'stop': - next_video[0] = default_uri - else: - next_video[0] = uri - playbin.seek( - 1.0, Gst.Format.TIME, Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT, - Gst.SeekType.END, 0, Gst.SeekType.NONE, 0) - - while True: - uri = sys.stdin.readline() - if uri == '': - break - elif len(uri.strip()) > 0: - set_uri(uri.strip()) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/frameobject/with-dialog2.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/frameobject/with-dialog2.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/frameobject/with-dialog-different-background.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/frameobject/with-dialog-different-background.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/frameobject/with-dialog.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/frameobject/with-dialog.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/frameobject/without-dialog-different-background.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/frameobject/without-dialog-different-background.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/frameobject/without-dialog.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/frameobject/without-dialog.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-dots-1px-border.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-dots-1px-border.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-dots.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-dots.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-red-columns.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-red-columns.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-red-columns-transparent.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-red-columns-transparent.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-red-rows.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-red-rows.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/blue-red-rows-transparent.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/blue-red-rows-transparent.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/frame.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/frame.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-blue-columns.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-blue-columns.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-blue-columns-transparent.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-blue-columns-transparent.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-blue-rows.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-blue-rows.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-blue-rows-transparent.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-blue-rows-transparent.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-dots-1px-border.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-dots-1px-border.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/region/red-dots.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/region/red-dots.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/badpyramid-frame2.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/badpyramid-frame2.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/badpyramid-frame.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/badpyramid-frame.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/badpyramid-reference2.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/badpyramid-reference2.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/badpyramid-reference.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/badpyramid-reference.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/roku-tile-frame.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/roku-tile-frame.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/images/regression/roku-tile-selection.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/images/regression/roku-tile-selection.png differ Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/ocr/00.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/ocr/00.png differ diff -Nru stb-tester-30-5-gbefe47c/tests/power-switch-stress-test.sh stb-tester-31/tests/power-switch-stress-test.sh --- stb-tester-30-5-gbefe47c/tests/power-switch-stress-test.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/power-switch-stress-test.sh 2019-09-18 14:04:32.000000000 +0000 @@ -6,8 +6,8 @@ #/ #/ Device URI is read from the stbt config file; the address of the individual #/ power outlet is replaced with the ones specified as command line arguments. -#/ Example: `power-switch-stress-test.sh 1-A{1-8}` sends requests to switch all -#/ 8 outlets of a PDUeX KWX unit simultaneously. +#/ Example: `power-switch-stress-test.sh 1-A{1..8}` sends requests to switch +#/ all 8 outlets of a PDUeX KWX unit simultaneously. set -u Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/tests/press_and_wait_visualisation.png and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/tests/press_and_wait_visualisation.png differ diff -Nru stb-tester-30-5-gbefe47c/tests/pylint.conf stb-tester-31/tests/pylint.conf --- stb-tester-30-5-gbefe47c/tests/pylint.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/pylint.conf 2019-09-18 14:04:32.000000000 +0000 @@ -1,2 +1,5 @@ -# Dummy pylintrc file so that the user's ~/.pylintrc configuration doesn't -# interfere with the tests in test-pylint-checker.sh. +# pylint configuration for the tests in test-stbt-lint.sh. + +[MESSAGES CONTROL] +disable= + superfluous-parens diff -Nru stb-tester-30-5-gbefe47c/tests/run_performance_test.py stb-tester-31/tests/run_performance_test.py --- stb-tester-30-5-gbefe47c/tests/run_performance_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/run_performance_test.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,10 @@ #!/usr/bin/python +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import glob import os import subprocess @@ -26,7 +31,7 @@ done >&2 """, shell=True) - print "screenshot,reference,min,avg,max" + print("screenshot,reference,min,avg,max") for fname in glob.glob("images/performance/*-frame.png"): tname = fname.replace("-frame.png", "-reference.png") @@ -34,11 +39,11 @@ t = stbt.load_image(tname) # pylint:disable=cell-var-from-loop times = timeit.repeat(lambda: stbt.match(t, f), number=1, repeat=100) - print "%s,%s,%f,%f,%f" % (os.path.basename(fname), + print("%s,%s,%f,%f,%f" % (os.path.basename(fname), os.path.basename(tname), min(times), max(times), - sum(times) / len(times)) + sum(times) / len(times))) if __name__ == "__main__": diff -Nru stb-tester-30-5-gbefe47c/tests/run-tests.sh stb-tester-31/tests/run-tests.sh --- stb-tester-30-5-gbefe47c/tests/run-tests.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/run-tests.sh 2019-09-18 14:04:32.000000000 +0000 @@ -7,24 +7,29 @@ #/ -i Run against installed version of stbt #/ -l Leave the scratch dir created in /tmp. #/ -v Verbose (don't suppress console output from tests). +#/ -x Stop on first failure. #/ #/ If any test names are specified, only those test cases will be run. -while getopts "lvi" option; do +while getopts "ilvx" option; do case $option in i) test_the_installed_version=true;; l) leave_scratch_dir=true;; v) verbose=true;; + x) stop_on_first_failure=true;; *) grep '^#/' < "$0" | cut -c4- >&2; exit 1;; # Print usage message esac done shift $(($OPTIND-1)) export testdir="$(cd "$(dirname "$0")" && pwd)" -export srcdir="$testdir/.." +export srcdir=$(realpath --no-symlinks "$testdir/..") +export LANG=C.UTF-8 export PYTHONUNBUFFERED=x export PYLINTRC="$testdir/pylint.conf" +export python_version=${python_version:=2.7} +export python=python$python_version testsuites=() testcases=() @@ -47,7 +52,7 @@ { echo "run-tests.sh: error: Failed to install stbt" >&2; exit 2; } export PATH="$test_installation_prefix/bin:$PATH" \ GST_PLUGIN_PATH=$test_installation_prefix/lib/gstreamer-1.0/plugins:$$GST_PLUGIN_PATH \ - PYTHONPATH=$test_installation_prefix/lib/python2.7/site-packages:$PYTHONPATH + PYTHONPATH=$test_installation_prefix/lib/python$python_version/site-packages:$PYTHONPATH fi . $testdir/utils.sh @@ -60,7 +65,8 @@ unset STBT_CONFIG_FILE cp "$testdir/stbt.conf" "$scratchdir/config/stbt" printf "$(bold $1...) " - ( cd "$scratchdir" && $1 ) > "$scratchdir/log" 2>&1 + ( cd "$scratchdir" && $1 ) > "$scratchdir/log" 2>&1 & + wait $! local status=$? case $status in 0) echo "$(green OK)";; @@ -87,6 +93,9 @@ ret=0 for t in ${testcases[*]}; do run $t || ret=1 + if [[ "$stop_on_first_failure" == "true" && $ret -eq 1 ]]; then + break + fi done if [[ -n "$test_installation_prefix" ]]; then rm -rf "$test_installation_prefix" diff -Nru stb-tester-30-5-gbefe47c/tests/stbt.conf stb-tester-31/tests/stbt.conf --- stb-tester-30-5-gbefe47c/tests/stbt.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/stbt.conf 2019-09-18 14:04:32.000000000 +0000 @@ -1,10 +1,7 @@ [global] source_pipeline = videotestsrc is-live=true ! video/x-raw,format=BGR,width=320,height=240,framerate=10/1 sink_pipeline = -transformation_pipeline = identity control = test -restart_source = False -source_teardown_eos = False verbose = 1 power_outlet = none @@ -16,9 +13,6 @@ should_be_true = True should_be_false = False -[camera] -video_format = mp4 - [match] match_method=sqdiff match_threshold=0.98 @@ -49,11 +43,5 @@ [run] save_video = -[batch] -pre_run = -post_run = -classify = -recover = - [special] test_key = not the global value diff -Nru stb-tester-30-5-gbefe47c/tests/subdirectory/test_load_image_from_subdirectory.py stb-tester-31/tests/subdirectory/test_load_image_from_subdirectory.py --- stb-tester-30-5-gbefe47c/tests/subdirectory/test_load_image_from_subdirectory.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/subdirectory/test_load_image_from_subdirectory.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import cv2 diff -Nru stb-tester-30-5-gbefe47c/tests/test2.py stb-tester-31/tests/test2.py --- stb-tester-30-5-gbefe47c/tests/test2.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test2.py 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -pass diff -Nru stb-tester-30-5-gbefe47c/tests/test_android.py stb-tester-31/tests/test_android.py --- stb-tester-30-5-gbefe47c/tests/test_android.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_android.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,8 @@ from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import re @@ -92,7 +96,7 @@ native_x, native_y = _to_native_coordinates( screenshot_x, screenshot_y, coordinate_system, _Dimensions(*device_resolution)) - print(native_x, native_y) + print((native_x, native_y)) assert isclose(native_x, expected_coordinates[0], atol=1) assert isclose(native_y, expected_coordinates[1], atol=1) diff -Nru stb-tester-30-5-gbefe47c/tests/test-camera.sh stb-tester-31/tests/test-camera.sh --- stb-tester-30-5-gbefe47c/tests/test-camera.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-camera.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,309 +0,0 @@ -# Run with ./run-tests.sh - -skip_if_no_rsvg_plugins() { - if ! gst-inspect-1.0 rsvg 2>&1 >/dev/null; then - skip "rsvg GStreamer plugins not installed" - fi -} -skip_if_no_stbt_camera() { - if ! stbt --with-experimental camera -h &>/dev/null; then - skip "stbt camera is not installed" - fi -} -skip_if_no_stbt_plugins() { - if ! gst-inspect-1.0 stbt &>/dev/null; then - skip "stbt GStreamer plugins not installed" - fi -} - -stb_tester_logo_src_1080p="\ - videotestsrc pattern=solid-color \ - ! video/x-raw,width=1280,height=720 \ - ! rsvgoverlay location=$testdir/stb-tester-350px.svg x=465 y=210 \ - ! videoscale \ - ! video/x-raw,width=1920,height=1080 \ - ! videoconvert " - -### -### stbtgeometriccorrection tests -### - -create_stb_tester_logo_template() { - # Can't just use rsvgdec as it doesn't seem to respect the white background - # colour in the svg - gst-launch-1.0 videotestsrc pattern=solid-color \ - ! video/x-raw,width=349,height=301 \ - ! rsvgoverlay location="$testdir/stb-tester-350px.svg" \ - ! videoconvert ! video/x-raw,format=RGB ! pngenc snapshot = true \ - ! filesink location="stb-tester-350px.png" -} - -test_that_stbtgeometriccorrection_scales_by_default() { - skip_if_no_stbt_plugins - skip_if_no_rsvg_plugins - - start_fake_video_src_launch_1080 $stb_tester_logo_src_1080p && - set_config global.transformation_pipeline "stbtgeometriccorrection" && - set_config global.control "none" && - - create_stb_tester_logo_template && - echo 'wait_for_match("stb-tester-350px.png")' >test.py && - stbt run -v test.py -} - -# Properties to be passed to stbtgeometriccorrection to flatten capture-logo.png. -# capture-logo.png was taken with a Logitech C920 webcam. -wp_matricies=' - camera-matrix="1491.1536435672558 0.0 929.63729425798135 - 0.0 1490.0565740887305 569.55885903330557 - 0.0 0.0 1.0" - - distortion-coefficients="0.12152211775145583 -0.28102519335279752 - 0.00020128754517049412 3.738779032027093e-05 0.08124443207970744" - - inv-homography-matrix="0.00078482598991913902 -3.0756177190069151e-05 -0.48720520841063641 - 3.731138534585565e-05 0.00078626284743211832 -0.33698755050692453 - -7.8645779307294576e-05 6.6181312307411495e-05 1.0236834039817024"' -wp_props="$(echo "$wp_matricies" | tr '\n' ' ')" - -test_that_stbtgeometriccorrection_flattens_pictures_of_TVs() { - skip_if_no_stbt_plugins - skip_if_no_rsvg_plugins - - create_stb_tester_logo_template && - start_fake_video_src_launch_1080 uridecodebin "uri=file://$testdir/capture-logo.png" ! videoconvert ! imagefreeze && - set_config global.transformation_pipeline "stbtgeometriccorrection $wp_props" && - set_config global.control "none" && - - create_stb_tester_logo_template && - echo 'wait_for_match("stb-tester-350px.png", - match_parameters=MatchParameters(confirm_threshold=0.7))' >test.py && - stbt run -v test.py -} - -test_that_stbt_camera_calibrate_corrects_for_geometric_distortion() { - skip_if_no_stbt_camera - skip_if_no_rsvg_plugins - - set_config camera.tv_driver assume - set_config global.control none - - start_fake_video_src_launch_1080 \ - uridecodebin "uri=file://$testdir/capture-chessboard.png" \ - ! videoconvert ! imagefreeze && - - stbt --with-experimental camera calibrate --noninteractive --skip-illumination && - start_fake_video_src_launch_1080 \ - uridecodebin "uri=file://$testdir/capture-letters-bw.png" \ - ! videoconvert ! imagefreeze && - stbt --with-experimental camera validate --positions-only letters-bw -} - -### -### Fake Video Source - test infrastructure. -### - -fake_video_src_source="\ - shmsrc do-timestamp=true is-live=true blocksize=2764800 \ - socket-path=gst-shm-socket ! \ - video/x-raw,format=RGB,width=1280,height=720,framerate=25/1 ! queue ! \ - videoconvert" - -start_fake_video_src() { - if [ -n "$FAKE_VIDEO_SRC_PID" ]; then - echo "Restarting Fake video src" - stop_fake_video_src - else - echo "Starting Fake video src" - fi - - rm -f gst-shm-socket - mkfifo uri_playlist - while cat uri_playlist; do true; done | \ - PYTHONPATH=$testdir/..:$PYTHONPATH "$testdir/fake-video-src.py" "$PWD/gst-shm-socket" & - FAKE_VIDEO_SRC_PID=$! - trap stop_fake_video_src EXIT - while [ ! -e gst-shm-socket ]; do - sleep 0.1 - done - if [ -n "$1" ]; then - overlay="rsvgoverlay location=$1" - else - overlay="identity" - fi - - set_config global.source_pipeline "shmsrc \ - do-timestamp=true is-live=true blocksize=2764800 \ - socket-path=$PWD/gst-shm-socket ! \ - video/x-raw,format=RGB,width=1280,height=720,framerate=25/1 ! \ - videoconvert ! $overlay ! videoscale ! videoconvert ! \ - video/x-raw,format=RGB,width=1920,height=1080" - - set_config camera.tv_driver fake:uri_playlist -} - -start_fake_video_src_launch() -{ - frame_bytes="$(expr $WIDTH \* $HEIGHT \* 3)" - shm_size="$(expr $frame_bytes \* 1000)" - rm -f gst-shm-socket - gst-launch-1.0 "$@" \ - ! video/x-raw,format=RGB,width=$WIDTH,height=$HEIGHT,framerate=25/1 \ - ! queue ! shmsink "wait-for-connection=true" "shm-size=$shm_size" \ - socket-path=$PWD/gst-shm-socket blocksize=$frame_bytes sync=true \ - buffer-time=100000000 & - FAKE_VIDEO_SRC_PID=$! - trap stop_fake_video_src EXIT - while [ ! -e gst-shm-socket ]; do - sleep 0.1 - done - - set_config global.source_pipeline \ - "shmsrc do-timestamp=true is-live=true blocksize=$frame_bytes \ - socket-path=$PWD/gst-shm-socket ! \ - video/x-raw,format=RGB,width=$WIDTH,height=$HEIGHT,framerate=25/1" -} - -start_fake_video_src_launch_720() -{ - WIDTH=1280 - HEIGHT=720 - start_fake_video_src_launch "$@" -} - -start_fake_video_src_launch_1080() -{ - WIDTH=1920 - HEIGHT=1080 - start_fake_video_src_launch "$@" -} - -fake_video_src_show() { - echo "$1" >uri_playlist -} - -stop_fake_video_src() { - kill "$FAKE_VIDEO_SRC_PID" - unset FAKE_VIDEO_SRC_PID - rm -f uri_playlist gst-shm-socket - true -} - -### -### stbt camera validate tests -### - -run_validation() { - color="$1" - extra=${2:-identity} - - skip_if_no_stbt_camera - skip_if_no_rsvg_plugins - - start_fake_video_src_launch_720 filesrc location="$testdir/$color.png" ! pngdec \ - ! videoconvert ! $extra ! videoconvert \ - ! video/x-raw,width=1280,height=720 ! imagefreeze && - set_config global.control none && - stbt --with-experimental camera validate --tv-driver=assume "$color" -} - -test_that_validation_passes_on_pristine_input() { - run_validation letters-bw || fail "Validation failed on pristine input" -} -test_that_validation_fails_if_letters_are_offset() { - run_validation letters-bw "videobox top=-2 left=-2 ! \ - videobox autocrop=true" \ - && fail "Validation succeeded on invalid input" - return 0 -} -test_that_validation_fails_if_letters_are_scaled_down() { - run_validation letters-bw "videoscale ! video/x-raw,width=1278,height=718 ! \ - videobox autocrop=true" \ - && fail "Validation succeeded on invalid input" - return 0 -} -test_that_validation_fails_if_letters_are_scaled_up() { - run_validation letters-bw "videoscale ! video/x-raw,width=1282,height=722 ! \ - videobox autocrop=true" \ - && fail "Validation succeeded on invalid input" - return 0 -} -test_that_validation_fails_with_vignetting() { - run_validation letters-bw \ - "rsvgoverlay location=$testdir/vignette-overlay.svg" \ - && fail "Validation succeeded on invalid input" - return 0 -} - -# Test manual driver - -test_that_validation_video_served_over_http_is_correct() { - # We test a lot of functionality in this test. Arguably it should be split - # down. We test that: - # - # * HTTP URLs are provided. - # * Videos can be played from those URLs. - # * The videos are what stbt camera validate was expecting and thus that - # the videos have been generated on demand successfully. - # * The validation code is hooked up to the drivers. - # * The manual driver responds to user input (pressing ). - # - # The setup looks like: - # - # |-----------------| video |----------------------| - # | fakevideosrc.py | --- gst-shm-socket --> | stbt camera validate | - # |-----------------| |----------------------| - # ^ stdin ^ stdin | stderr - # | | instructions - # | stbt_validate_input | - # | | V - # | |----------------------| - # uri_playlist ----------------------------| this test | - # |----------------------| - # - # The test listens for instructions from stbt camera validate's stderr and - # instructs fakevideosrc.py to display URIs before telling stbt camera - # validate to proceed by pressing enter on it's stdin - - skip_if_no_stbt_camera - skip_if_no_rsvg_plugins - - start_fake_video_src - set_config global.control none - set_config global.transformation_pipeline "videoscale ! video/x-raw,width=1280,height=720 ! identity" - - mkfifo stbt_validate_input - while cat stbt_validate_input; do true; done | \ - stbt --with-experimental camera validate --tv-driver=manual 2>&1 | (\ - while read line; do - if [[ "$line" =~ 'http://' ]]; then - fake_video_src_show "$(echo "$line" | grep -Eo 'http://\S*')" - fi - if [[ "$line" =~ 'Press ' ]]; then - sleep 1 - echo >stbt_validate_input - fi - done - - # Have to wake the cat up once more to get it to tear itself down - # properly: - mv stbt_validate_input stbt_validate_input.teardown - echo "TEARDOWN" >stbt_validate_input.teardown - rm stbt_validate_input.teardown - ) -} - -# Test illumination compensation - -test_illumination_compensation() { - skip_if_no_stbt_camera - skip_if_no_rsvg_plugins - - export STBT_TEST_VALIDATION_WAIT_TIMEOUT=60 - - set_config global.control none - start_fake_video_src "$testdir/vignette-overlay.svg" && - stbt --with-experimental camera calibrate --noninteractive && - start_fake_video_src "$testdir/vignette-overlay.svg" && - stbt --with-experimental camera validate -} diff -Nru stb-tester-30-5-gbefe47c/tests/test_config.py stb-tester-31/tests/test_config.py --- stb-tester-30-5-gbefe47c/tests/test_config.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_config.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import enum import os from contextlib import contextmanager diff -Nru stb-tester-30-5-gbefe47c/tests/test_core.py stb-tester-31/tests/test_core.py --- stb-tester-30-5-gbefe47c/tests/test_core.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_core.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,11 @@ # coding: utf-8 +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + import itertools import os import shutil @@ -31,7 +37,7 @@ def test_load_image_with_unicode_filename(): - print sys.getfilesystemencoding() + print(sys.getfilesystemencoding()) shutil.copyfile(_find_file("Rothlisberger.png"), _find_file("Röthlisberger.png")) assert stbt.load_image("Röthlisberger.png") is not None @@ -56,6 +62,24 @@ stbt.crop(f, stbt.Region(x=1045, y=672, right=1281, bottom=721)) +def test_region_intersect(): + r1 = stbt.Region(0, 0, right=20, bottom=10) + r2 = stbt.Region(5, 5, right=25, bottom=15) + expected = stbt.Region(5, 5, right=20, bottom=10) + assert expected == stbt.Region.intersect(r1, r2) + with pytest.raises(AttributeError): + r1.intersect(r2) # pylint:disable=no-member + + +def test_region_bounding_box(): + r1 = stbt.Region(0, 0, right=20, bottom=10) + r2 = stbt.Region(5, 5, right=25, bottom=15) + expected = stbt.Region(0, 0, right=25, bottom=15) + assert expected == stbt.Region.bounding_box(r1, r2) + with pytest.raises(AttributeError): + r1.bounding_box(r2) # pylint:disable=no-member + + def test_region_replace(): r = stbt.Region(x=10, y=20, width=20, height=30) @@ -189,7 +213,7 @@ class Zero(object): - def __nonzero__(self): + def __bool__(self): return False def __eq__(self, other): diff -Nru stb-tester-30-5-gbefe47c/tests/test_error.py stb-tester-31/tests/test_error.py --- stb-tester-30-5-gbefe47c/tests/test_error.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_error.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -import stbt - -if __name__ == '__main__': - raise stbt.UITestError("Test Error") diff -Nru stb-tester-30-5-gbefe47c/tests/test_failure.py stb-tester-31/tests/test_failure.py --- stb-tester-30-5-gbefe47c/tests/test_failure.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_failure.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -import stbt - -if __name__ == '__main__': - raise stbt.UITestFailure("Test Failure") diff -Nru stb-tester-30-5-gbefe47c/tests/test_frameobject.py stb-tester-31/tests/test_frameobject.py --- stb-tester-30-5-gbefe47c/tests/test_frameobject.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_frameobject.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import threading import stbt @@ -27,7 +32,7 @@ False >>> fo FalseyFrameObject(is_visible=False) - >>> print fo.public + >>> print(fo.public) None >>> fo._private 6 @@ -96,7 +101,7 @@ >>> bigred = OrderedFrameObject(numpy.array([[[0, 0, 255], [0, 0, 255]]])) >>> green = OrderedFrameObject(numpy.array([[[0, 255, 0]]])) >>> blue = OrderedFrameObject(numpy.array([[[255, 0, 0]]])) - >>> print sorted([red, green, blue, bigred]) + >>> print(sorted([red, green, blue, bigred])) [...'blue'..., ...'green'..., ...'red', size=1..., ...'red', size=2)] """ @@ -143,17 +148,17 @@ """ @property def is_visible(self): - print "is_visible called" + print("is_visible called") return self._helper @property def _helper(self): - print "_helper called" + print("_helper called") return 7 @property def another(self): - print "another called" + print("another called") return self._helper + 3 @@ -170,9 +175,9 @@ False >>> m.is_visible False - >>> print m.public + >>> print(m.public) None - >>> print m.another + >>> print(m.another) None >>> m._private 7 @@ -186,29 +191,29 @@ """ @property def is_visible(self): - print "is_visible called" + print("is_visible called") ten = self.public seven = self._private return bool(ten < seven) @property def _private(self): - print "_private called" + print("_private called") return 7 @property def public(self): - print "public called" + print("public called") return self._private + 3 @property def another(self): - print "another called" + print("another called") return 10 @property def _another(self): - print "_another called" + print("_another called") return 11 @@ -228,21 +233,12 @@ t.start() for t in threads: t.join() - print results + print(results) assert results == {n: None for n in range(10)} def _load_frame(name): - import cv2 - from os.path import abspath, dirname - filename = "%s/%s/frame-object-%s.png" % ( - dirname(abspath(__file__)), - "auto-selftest-example-test-pack/selftest/screenshots", - name) - frame = cv2.imread(filename) - if frame is None: - raise ValueError("Couldn't load test image %r" % filename) - return frame + return stbt.load_image("images/frameobject/%s.png" % name) class Dialog(stbt.FrameObject): @@ -256,9 +252,9 @@ Some basic operations: - >>> print dialog.message + >>> print(dialog.message) This set-top box is great - >>> print dialog_fab.message + >>> print(dialog_fab.message) This set-top box is fabulous ``FrameObject`` defines truthiness of your objects based on the mandatory @@ -272,7 +268,7 @@ If ``is_visible`` is falsey, all the rest of the properties will be ``None``: - >>> print no_dialog.message + >>> print(no_dialog.message) None This enables usage like:: @@ -312,8 +308,8 @@ ``FrameObject`` defines ``__hash__`` too so you can store them in a set or in a dict: - >>> {dialog} - set([Dialog(is_visible=True, message=u'This set-top box is great', title=u'Information')]) + >>> {dialog: 1} + {Dialog(is_visible=True, message=u'This set-top box is great', title=u'Information'): 1} >>> len({no_dialog, dialog, dialog, dialog_bunnies}) 2 @@ -326,10 +322,10 @@ ``refresh`` returns a new FrameObject of the same type: >>> page = Dialog(frame=_load_frame('with-dialog')) - >>> print page.message + >>> print(page.message) This set-top box is great >>> page = page.refresh(_load_frame('with-dialog2')) - >>> print page.message + >>> print(page.message) This set-top box is fabulous """ diff -Nru stb-tester-30-5-gbefe47c/tests/test_functions.py stb-tester-31/tests/test_functions.py --- stb-tester-30-5-gbefe47c/tests/test_functions.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_functions.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -import os - - -def test_that_this_test_is_run(): - open("touched", "w").close() - - -def test_that_does_nothing(): - pass - - -def test_that_asserts_the_impossible(): - assert 1 + 1 == 3 - - -def test_that_chdirs(): - os.chdir("/tmp") - - -def test_that_dumps_core(): - os.abort() diff -Nru stb-tester-30-5-gbefe47c/tests/test_grid.py stb-tester-31/tests/test_grid.py --- stb-tester-30-5-gbefe47c/tests/test_grid.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/tests/test_grid.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,141 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +from itertools import combinations + +import networkx as nx +from pytest import raises + +from stbt import Grid, grid_to_navigation_graph, Position, Region + + +def test_grid(): + g = Grid(Region(0, 0, 6, 2), cols=6, rows=2) + assert g.area == 12 + assert len(g) == 12 + + def check_conversions(g, region, position, index): + c = g.get(index=index) + assert c.region == region + assert c.position == position + assert c.data is None + assert c == g.get(region=region) + assert c == g.get(position=position) + assert c == g[index] + assert c == g[position] + assert c == g[region] + + check_conversions(g, Region(0, 0, 1, 1), (0, 0), 0) + check_conversions(g, Region(5, 1, 1, 1), (5, 1), 11) + check_conversions(g, Region(5, 1, 1, 1), Position(5, 1), 11) + + assert g.get(region=Region(4, 0, 3, 3)).position == (5, 1) + for x, y in [(-1, 0), (0, -1), (6, 0), (0, 2), (6, 2)]: + with raises(IndexError): + g.get(region=Region(x, y, 1, 1)) + + with raises(IndexError): + g.get(index=12) + with raises(IndexError): + g.get(index=-13) + with raises(IndexError): + g.get(position=(6, 1)) + with raises(IndexError): + g.get(data="J") + + g = Grid(Region(x=99, y=212, width=630, height=401), cols=5, rows=3) + check_conversions(g, Region(351, 212, 126, 133), (2, 0), 2) + check_conversions(g, Region(477, 345, 126, 134), (3, 1), 8) + + # If you use a region from a different source (e.g. stbt.match) then the + # region you get *back* from the Grid should be the region defined by the + # grid. + r = Region(x=99, y=212, width=126, height=133) + assert r == g.get(region=r.extend(right=5, bottom=5)).region + + for r1, r2 in combinations(g.cells, 2): + assert Region.intersect(r1.region, r2.region) is None + + for i, c in enumerate(g): + assert i == c.index + + +def test_grid_with_data(): + layout = ["ABCDEFG", + "HIJKLMN", + "OPQRSTU", + "VWXYZ-'"] + g = Grid(Region(0, 0, 100, 50), data=layout) + assert g.cols == 7 + assert g.rows == 4 + assert g.get(index=9).data == "J" + assert g.get(position=Position(x=2, y=1)).data == "J" + assert g.get(data="J").index == 9 + assert g["J"].index == 9 + assert g[Position(x=2, y=1)].data == "J" + assert g[2, 1].data == "J" + assert g[-1].data == "'" + for x in ["a", layout[0], layout]: + with raises(IndexError): + print(g[x]) + + +def test_grid_to_navigation_graph(): + grid = Grid(region=None, data=["ABC", + "DEF"]) + graph = grid_to_navigation_graph(grid) + expected = nx.parse_edgelist( + """ + A B KEY_RIGHT + A D KEY_DOWN + B A KEY_LEFT + B C KEY_RIGHT + B E KEY_DOWN + C B KEY_LEFT + C F KEY_DOWN + D A KEY_UP + D E KEY_RIGHT + E B KEY_UP + E D KEY_LEFT + E F KEY_RIGHT + F C KEY_UP + F E KEY_LEFT + """.split("\n"), + create_using=nx.DiGraph(), + data=[("key", str)]) + assert sorted(expected.edges(data=True)) == sorted(graph.edges(data=True)) + assert graph["A"]["B"] == {"key": "KEY_RIGHT"} + assert graph["B"] == {"A": {"key": "KEY_LEFT"}, + "C": {"key": "KEY_RIGHT"}, + "E": {"key": "KEY_DOWN"}} + + +def test_grid_to_navigation_graph_without_data(): + # 012 + # 345 + grid = Grid(region=None, cols=3, rows=2) + graph = grid_to_navigation_graph(grid) + expected = nx.parse_edgelist( + """ + 0 1 KEY_RIGHT + 0 3 KEY_DOWN + 1 0 KEY_LEFT + 1 2 KEY_RIGHT + 1 4 KEY_DOWN + 2 1 KEY_LEFT + 2 5 KEY_DOWN + 3 0 KEY_UP + 3 4 KEY_RIGHT + 4 1 KEY_UP + 4 3 KEY_LEFT + 4 5 KEY_RIGHT + 5 2 KEY_UP + 5 4 KEY_LEFT + """.split("\n"), + create_using=nx.DiGraph(), + nodetype=int, + data=[("key", str)]) + assert sorted(expected.edges(data=True)) == sorted(graph.edges(data=True)) diff -Nru stb-tester-30-5-gbefe47c/tests/test_irnetbox_proxy.py stb-tester-31/tests/test_irnetbox_proxy.py --- stb-tester-30-5-gbefe47c/tests/test_irnetbox_proxy.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_irnetbox_proxy.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -import imp -import os -import unittest - -IRNetBoxProxy = imp.load_source( - "irnetbox_proxy", - os.path.join(os.path.dirname(__file__), "..", "irnetbox-proxy") -).IRNetBoxProxy - - -class ProxyTest(unittest.TestCase): - - def test_id_generation(self): - proxy = IRNetBoxProxy("no_address") - for i in range(65536): - self.assertEqual(proxy.make_id(), i) - self.assertEqual(proxy.make_id(), 0) - - def test_id_generation_skips_active_ids(self): - proxy = IRNetBoxProxy("no_address") - proxy.async_commands[2] = None - - self.assertEqual(proxy.make_id(), 0) - self.assertEqual(proxy.make_id(), 1) - self.assertEqual(proxy.make_id(), 3) - - def test_replace_sequence(self): - proxy = IRNetBoxProxy("no_address") - data = "YYYY" - self.assertEqual(proxy.replace_sequence_id(data, 1), "\x00\x01YY") - self.assertEqual(proxy.replace_sequence_id(data, 65535), "\xff\xffYY") - - -if __name__ == "__main__": - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/tests/test-irnetbox.sh stb-tester-31/tests/test-irnetbox.sh --- stb-tester-30-5-gbefe47c/tests/test-irnetbox.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-irnetbox.sh 2019-09-18 14:04:32.000000000 +0000 @@ -6,7 +6,8 @@ return 1 } - PYTHONPATH=$srcdir "$testdir"/fake-irnetbox "$@" > fake-irnetbox.log & + PYTHONPATH=$srcdir $python "$testdir"/fake-irnetbox "$@" \ + > fake-irnetbox.log & fake_irnetbox=$! trap "kill $fake_irnetbox" EXIT waitfor "^PORT=" fake-irnetbox.log || fail "fake-irnetbox failed to start" @@ -24,7 +25,7 @@ ir.irsend_raw(port=1, power=100, data=rcu["MENU"]) ir.irsend_raw(port=1, power=100, data=rcu["OK"]) EOF - PYTHONPATH=$srcdir python test.py || return + PYTHONPATH=$srcdir $python test.py || return grep -q "Received message POWER_ON" fake-irnetbox.log || fail "fake-irnetbox didn't receive POWER_ON message" [[ "$(grep "Received message OUTPUT_IR_ASYNC" fake-irnetbox.log | @@ -78,9 +79,9 @@ cat > test.py <<-EOF import time - print "Before press: %d" % time.time() + print("Before press: %d" % time.time()) press("MENU") - print "After press: %d" % time.time() + print("After press: %d" % time.time()) EOF stbt run -v \ --control irnetbox:localhost:$irnetbox_port:1:"$testdir"/irnetbox.conf \ @@ -103,40 +104,3 @@ test.py || fail "Expected 'press' to raise exception" cat log | grep -q timeout || fail "Didn't raise timeout" } - -test_irnetbox_proxy() { - export PYTHONPATH=$srcdir - - start_fake_irnetbox - proxy_port=5887 - "$srcdir"/irnetbox-proxy -vv \ - -l localhost -p $proxy_port \ - localhost $irnetbox_port & - proxy=$! - trap "kill $fake_irnetbox $proxy" EXIT - - cat > test1.py <<-EOF - import time - from _stbt import irnetbox - with irnetbox.IRNetBox("localhost", $proxy_port) as ir: - for _ in range(10): - ir.indicators_on() - print "test1.py: Sent CPLD_INSTRUCTION" - time.sleep(0.1) - EOF - - cat > test2.py <<-EOF - import time - from _stbt import irnetbox - rcu = irnetbox.RemoteControlConfig("$testdir/irnetbox.conf") - with irnetbox.IRNetBox("localhost", $proxy_port) as ir: - for _ in range(10): - ir.irsend_raw(port=1, power=100, data=rcu["MENU"]) - print "test2.py: Sent OUTPUT_IR_ASYNC" - time.sleep(0.1) - EOF - - python test1.py & test1=$! - python test2.py & test2=$! - wait $test1 $test2 || return -} diff -Nru stb-tester-30-5-gbefe47c/tests/test_keyboard.py stb-tester-31/tests/test_keyboard.py --- stb-tester-30-5-gbefe47c/tests/test_keyboard.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/tests/test_keyboard.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,425 @@ +# coding: utf-8 + +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order + +import networkx as nx +import mock +import numpy +import pytest + +import stbt +from stbt.keyboard import _add_weights, _keys_to_press +from _stbt.transition import _TransitionResult, TransitionStatus + + +GRAPH = """ + A B KEY_RIGHT + A H KEY_DOWN + B A KEY_LEFT + B C KEY_RIGHT + B I KEY_DOWN + C B KEY_LEFT + C D KEY_RIGHT + C J KEY_DOWN + D C KEY_LEFT + D E KEY_RIGHT + D K KEY_DOWN + E D KEY_LEFT + E F KEY_RIGHT + E L KEY_DOWN + F E KEY_LEFT + F G KEY_RIGHT + F M KEY_DOWN + G F KEY_LEFT + G N KEY_DOWN + H I KEY_RIGHT + H A KEY_UP + H O KEY_DOWN + I H KEY_LEFT + I J KEY_RIGHT + I B KEY_UP + I P KEY_DOWN + J I KEY_LEFT + J K KEY_RIGHT + J C KEY_UP + J Q KEY_DOWN + K J KEY_LEFT + K L KEY_RIGHT + K D KEY_UP + K R KEY_DOWN + L K KEY_LEFT + L M KEY_RIGHT + L E KEY_UP + L S KEY_DOWN + M L KEY_LEFT + M N KEY_RIGHT + M F KEY_UP + M T KEY_DOWN + N M KEY_LEFT + N G KEY_UP + N U KEY_DOWN + O P KEY_RIGHT + O H KEY_UP + O V KEY_DOWN + P O KEY_LEFT + P Q KEY_RIGHT + P I KEY_UP + P W KEY_DOWN + Q P KEY_LEFT + Q R KEY_RIGHT + Q J KEY_UP + Q X KEY_DOWN + R Q KEY_LEFT + R S KEY_RIGHT + R K KEY_UP + R Y KEY_DOWN + S R KEY_LEFT + S T KEY_RIGHT + S L KEY_UP + S Z KEY_DOWN + T S KEY_LEFT + T U KEY_RIGHT + T M KEY_UP + T - KEY_DOWN + U T KEY_LEFT + U N KEY_UP + U ' KEY_DOWN + V W KEY_RIGHT + V O KEY_UP + V SPACE KEY_DOWN + W V KEY_LEFT + W X KEY_RIGHT + W P KEY_UP + W SPACE KEY_DOWN + X W KEY_LEFT + X Y KEY_RIGHT + X Q KEY_UP + X SPACE KEY_DOWN + Y X KEY_LEFT + Y Z KEY_RIGHT + Y R KEY_UP + Y SPACE KEY_DOWN + Z Y KEY_LEFT + Z - KEY_RIGHT + Z S KEY_UP + Z SPACE KEY_DOWN + - Z KEY_LEFT + - ' KEY_RIGHT + - T KEY_UP + - SPACE KEY_DOWN + ' - KEY_LEFT + ' U KEY_UP + ' SPACE KEY_DOWN + SPACE CLEAR KEY_RIGHT + SPACE V KEY_UP + SPACE W KEY_UP + SPACE X KEY_UP + SPACE Y KEY_UP + SPACE Z KEY_UP + SPACE - KEY_UP + SPACE ' KEY_UP + CLEAR SPACE KEY_LEFT + CLEAR SEARCH KEY_RIGHT + CLEAR V KEY_UP + CLEAR W KEY_UP + CLEAR X KEY_UP + CLEAR Y KEY_UP + CLEAR Z KEY_UP + CLEAR - KEY_UP + CLEAR ' KEY_UP + SEARCH CLEAR KEY_LEFT + SEARCH V KEY_UP + SEARCH W KEY_UP + SEARCH X KEY_UP + SEARCH Y KEY_UP + SEARCH Z KEY_UP + SEARCH - KEY_UP + SEARCH ' KEY_UP +""" +G = stbt.Keyboard.parse_edgelist(GRAPH) +nx.relabel_nodes(G, {"SPACE": " "}, copy=False) + + +def test_keys_to_press(): + assert list(_keys_to_press(G, "A", "A")) == [] + assert list(_keys_to_press(G, "A", "B")) == [("KEY_RIGHT", {"B"})] + assert list(_keys_to_press(G, "B", "A")) == [("KEY_LEFT", {"A"})] + assert list(_keys_to_press(G, "A", "C")) == [("KEY_RIGHT", {"B"}), + ("KEY_RIGHT", {"C"})] + assert list(_keys_to_press(G, "C", "A")) == [("KEY_LEFT", {"B"}), + ("KEY_LEFT", {"A"})] + assert list(_keys_to_press(G, "A", "H")) == [("KEY_DOWN", {"H"})] + assert list(_keys_to_press(G, "H", "A")) == [("KEY_UP", {"A"})] + assert list(_keys_to_press(G, "A", "I")) in ( + [("KEY_RIGHT", {"B"}), ("KEY_DOWN", {"I"})], + [("KEY_DOWN", {"H"}), ("KEY_RIGHT", {"I"})]) + assert list(_keys_to_press(G, " ", "A")) == [ + ("KEY_UP", {"V", "W", "X", "Y", "Z", "-", "'"})] + + +def test_add_weights(): + G = nx.parse_edgelist( # pylint:disable=redefined-outer-name + """ W SPACE KEY_DOWN + X SPACE KEY_DOWN + Y SPACE KEY_DOWN + Z SPACE KEY_DOWN + SPACE W KEY_UP + SPACE X KEY_UP + SPACE Y KEY_UP + SPACE Z KEY_UP + W X KEY_RIGHT + X Y KEY_RIGHT + Y Z KEY_RIGHT""".split("\n"), + create_using=nx.DiGraph(), + data=[("key", str)]) + + # This is the bug: + assert nx.shortest_path(G, "W", "Z") == ["W", "SPACE", "Z"] + + # And this is how we fix it: + _add_weights(G) + assert nx.shortest_path(G, "W", "Z", weight="weight") == [ + "W", "X", "Y", "Z"] + + +class _Keyboard(stbt.FrameObject): + """Immutable FrameObject representing the test's view of the Device Under + Test (``dut``). + + The keyboard looks like this:: + + A B C D E F G + H I J K L M N + O P Q R S T U + V W X Y Z - ' + SPACE CLEAR SEARCH + + """ + def __init__(self, dut): + super(_Keyboard, self).__init__( + frame=numpy.zeros((720, 1280, 3), dtype=numpy.uint8)) + self._dut = dut # Device Under Test -- i.e. ``YouTubeKeyboard`` + self._selection = self._dut.selection + + @property + def is_visible(self): + return True + + @property + def selection(self): + return self._selection + + def refresh(self, frame=None, **kwargs): + print("_Keyboard.refresh: Now on %r" % self._dut.selection) + return _Keyboard(dut=self._dut) + + KEYBOARD = stbt.Keyboard(GRAPH, navigate_timeout=0.1) + + def enter_text(self, text): + return self.KEYBOARD.enter_text(text.upper(), page=self) + + def navigate_to(self, target, verify_every_keypress=False): + return self.KEYBOARD.navigate_to( + target, page=self, verify_every_keypress=verify_every_keypress) + + +class YouTubeKeyboard(object): + """Fake keyboard implementation for testing.""" + + def __init__(self): + self.selection = "A" + self.page = _Keyboard(dut=self) + self.pressed = [] + self.entered = "" + # Pressing up from SPACE returns to the last letter we were at: + self.prev_state = "A" + + def press(self, key): + print("Pressed %s" % key) + self.pressed.append(key) + if key == "KEY_OK": + self.entered += self.selection + else: + next_states = [ + t for _, t, k in G.edges(self.selection, data="key") + if k == key] + if self.prev_state in next_states: + next_state = self.prev_state + else: + next_state = next_states[0] + if self.selection not in (" ", "CLEAR", "SEARCH"): + self.prev_state = self.selection + self.selection = next_state + + def press_and_wait(self, key, **kwargs): # pylint:disable=unused-argument + self.press(key) + return _TransitionResult(key, None, TransitionStatus.COMPLETE, 0, 0, 0) + + +class BuggyKeyboard(YouTubeKeyboard): + def press(self, key): + super(BuggyKeyboard, self).press(key) + if key == "KEY_RIGHT" and self.selection == "B": + self.selection = "C" + + +@pytest.fixture(scope="function") +def youtubekeyboard(): + kb = YouTubeKeyboard() + with mock.patch("stbt.press", kb.press), \ + mock.patch("stbt.press_and_wait", kb.press_and_wait): + yield kb + + +@pytest.fixture(scope="function") +def buggykeyboard(): + """Pressing KEY_RIGHT from A skips over B and lands on C. + + Note that the model we specify in our test-scripts still thinks that + KEY_RIGHT should land on B. This simulates a bug in the device-under-test, + not in the test-scripts. + """ + kb = BuggyKeyboard() + with mock.patch("stbt.press", kb.press), \ + mock.patch("stbt.press_and_wait", kb.press_and_wait): + yield kb + + +def test_enter_text(youtubekeyboard): # pylint:disable=redefined-outer-name + page = youtubekeyboard.page + assert page.selection == "A" + page = page.enter_text("hi there") + assert page.selection == "E" + assert youtubekeyboard.entered == "HI THERE" + + +def test_that_enter_text_uses_minimal_keypresses(youtubekeyboard): # pylint:disable=redefined-outer-name + page = youtubekeyboard.page + assert page.selection == "A" + page.enter_text("HI") + assert youtubekeyboard.pressed == ["KEY_DOWN", "KEY_OK", + "KEY_RIGHT", "KEY_OK"] + + +def test_that_keyboard_validates_the_targets(youtubekeyboard): # pylint:disable=redefined-outer-name + page = youtubekeyboard.page + with pytest.raises(ValueError): + page.enter_text("ABCÑ") + assert youtubekeyboard.pressed == [] + with pytest.raises(ValueError): + page.navigate_to("Ñ") + assert youtubekeyboard.pressed == [] + + +def test_navigate_to(youtubekeyboard): # pylint:disable=redefined-outer-name + page = youtubekeyboard.page + assert page.selection == "A" + page = page.navigate_to("SEARCH") + assert page.selection == "SEARCH" + assert youtubekeyboard.pressed == ["KEY_DOWN"] * 4 + ["KEY_RIGHT"] * 2 + + +@pytest.mark.parametrize("target,verify_every_keypress,num_presses", [ + ("B", False, 1), + ("B", True, 1), + ("C", False, 2), + ("C", True, 1), +]) +def test_that_navigate_to_checks_target( + buggykeyboard, target, verify_every_keypress, num_presses): # pylint:disable=redefined-outer-name + """buggykeyboard skips the B when pressing right from A (and lands on C).""" + page = buggykeyboard.page + assert page.selection == "A" + with pytest.raises(AssertionError): + page.navigate_to(target, verify_every_keypress) + assert buggykeyboard.pressed == ["KEY_RIGHT"] * num_presses + + +def test_composing_complex_keyboards(): + """The YouTube keyboard on Roku looks like this:: + + A B C D E F G + H I J K L M N + O P Q R S T U + V W X Y Z - ' + SPACE CLEAR SEARCH + + The first 4 rows behave normally within themselves. The bottom row behaves + normally within itself. But navigating to or from the bottom row is a bit + irregular: No matter what column you're in, when you press KEY_DOWN you + always land on SPACE. Then when you press KEY_UP, you go back to the column + you were last on -- even if you had pressed KEY_RIGHT/KEY_LEFT to move + within the bottom row. It's almost like they're two separate state + machines, and we can model them as such, with a few explicit connections + between the two. + """ + letters = stbt.Grid(stbt.Region(x=540, y=100, right=840, bottom=280), + data=["ABCDEFG", + "HIJKLMN", + "OPQRSTU", + "VWXYZ-'"]) + space_row = stbt.Grid(stbt.Region(x=540, y=280, right=840, bottom=330), + data=[[" ", "CLEAR", "SEARCH"]]) + + # Technique #0: Write the entire edgelist manually (as per previous tests) + K0 = stbt.Keyboard(GRAPH) + + # Technique #1: Manipulate the graph (manually or programmatically) directly + G1 = nx.compose(stbt.grid_to_navigation_graph(letters), + stbt.grid_to_navigation_graph(space_row)) + # Pressing down from the bottom row always goes to SPACE: + for k in letters.data[-1]: + G1.add_edge(k, " ", key="KEY_DOWN") + # Pressing back up from the space/clear/search row can go to any column + # in the bottom row: + for k in space_row.data[0]: + for j in letters.data[-1]: + G1.add_edge(k, j, key="KEY_UP") + K1 = stbt.Keyboard(G1) + + assert sorted(K0.G.edges(data=True)) == sorted(K1.G.edges(data=True)) + + # Technique #2: Use manually-written edgelist only for the irregular edges + # Note that Keyboard.__init__ will normalise "SPACE" -> " " so it doesn't + # matter if the 3 different graphs have different representations for + # "SPACE". + connections = stbt.Keyboard.parse_edgelist(""" + V SPACE KEY_DOWN + W SPACE KEY_DOWN + X SPACE KEY_DOWN + Y SPACE KEY_DOWN + Z SPACE KEY_DOWN + - SPACE KEY_DOWN + ' SPACE KEY_DOWN + SPACE V KEY_UP + SPACE W KEY_UP + SPACE X KEY_UP + SPACE Y KEY_UP + SPACE Z KEY_UP + SPACE - KEY_UP + SPACE ' KEY_UP + CLEAR V KEY_UP + CLEAR W KEY_UP + CLEAR X KEY_UP + CLEAR Y KEY_UP + CLEAR Z KEY_UP + CLEAR - KEY_UP + CLEAR ' KEY_UP + SEARCH V KEY_UP + SEARCH W KEY_UP + SEARCH X KEY_UP + SEARCH Y KEY_UP + SEARCH Z KEY_UP + SEARCH - KEY_UP + SEARCH ' KEY_UP + """) + G2 = nx.compose_all([stbt.grid_to_navigation_graph(letters), + stbt.grid_to_navigation_graph(space_row), + connections]) + K2 = stbt.Keyboard(G2) + + assert sorted(K0.G.edges(data=True)) == sorted(K2.G.edges(data=True)) diff -Nru stb-tester-30-5-gbefe47c/tests/test_lirc_control.py stb-tester-31/tests/test_lirc_control.py --- stb-tester-30-5-gbefe47c/tests/test_lirc_control.py 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/tests/test_lirc_control.py 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,125 @@ +import os +import subprocess +from collections import namedtuple +from textwrap import dedent + +import pytest + +from stbt import wait_until +from _stbt.control import uri_to_control +from _stbt.utils import named_temporary_directory, scoped_process + +# pylint:disable=redefined-outer-name + + +@pytest.yield_fixture(scope="function") +def lircd(): + with named_temporary_directory("stbt-lirc-test") as tmpdir: + socket = os.path.join(tmpdir, "lircd.socket") + logfile = os.path.join(tmpdir, "lircd.log") + proc = subprocess.Popen( + ["lircd", "--nodaemon", "--loglevel=info", "--logfile=/dev/stderr", + "--driver=file", "--device", logfile, # lircd output + "--output", socket, # lircd reads instructions from here + "--pidfile=%s/lircd.pid" % tmpdir, + _find_file("Apple_TV.lircd.conf")]) + wait_until(lambda: ( + os.path.exists(socket) or proc.poll() is not None)) + + with scoped_process(proc): + yield namedtuple("Lircd", "socket logfile")(socket, logfile) + + +def test_press(lircd): + control = uri_to_control("lirc:%s:Apple_TV" % lircd.socket) + control.press("KEY_OK") + lircd_output = open(lircd.logfile, "r").read() + expected = dedent("""\ + pulse 9000 + space 4500 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 527 + pulse 527 + space 527 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 1703 + pulse 527 + space 527 + pulse 527 + space 38000 + """) + assert expected == lircd_output + + +def test_press_with_unknown_remote(lircd): + control = uri_to_control("lirc:%s:roku" % lircd.socket) + with pytest.raises(RuntimeError) as excinfo: + control.press("KEY_OK") + assert 'unknown remote: "roku"' in str(excinfo.value) + + +def test_press_with_unknown_key(lircd): + control = uri_to_control("lirc:%s:Apple_TV" % lircd.socket) + with pytest.raises(RuntimeError) as excinfo: + control.press("KEY_MAGIC") + assert 'unknown command: "KEY_MAGIC"' in str(excinfo.value) + + +def _find_file(path, root=os.path.dirname(os.path.abspath(__file__))): + return os.path.join(root, path) diff -Nru stb-tester-30-5-gbefe47c/tests/test-lirc.sh stb-tester-31/tests/test-lirc.sh --- stb-tester-30-5-gbefe47c/tests/test-lirc.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-lirc.sh 2019-09-18 14:04:32.000000000 +0000 @@ -6,7 +6,7 @@ return 1 } - "$testdir"/fake-lircd "$@" > fake-lircd.log & + $python "$testdir"/fake-lircd "$@" > fake-lircd.log & fake_lircd=$! trap "kill $fake_lircd" EXIT waitfor "^SOCKET=" fake-lircd.log || fail "fake-lircd failed to start" @@ -21,8 +21,8 @@ press("ok") EOF stbt run -v --control lirc:$lircd_socket:test test.py || return - grep -q "fake-lircd: Received: SEND_ONCE test menu" fake-lircd.log && - grep -q "fake-lircd: Received: SEND_ONCE test ok" fake-lircd.log || + grep -Eq "fake-lircd: Received: b?'?SEND_ONCE test menu" fake-lircd.log && + grep -Eq "fake-lircd: Received: b?'?SEND_ONCE test ok" fake-lircd.log || fail "fake-lircd didn't receive 2 SEND_ONCE messages" } @@ -44,10 +44,10 @@ cat > test.py <<-EOF press("button_that_causes_timeout") EOF - timeout 10s stbt run -v --control lirc:$lircd_socket:test test.py + timeout 30s stbt run -v --control lirc:$lircd_socket:test test.py ret=$? - [ $? -eq $timedout ] && fail "'press' timed out" - [ $? -ne 0 ] || fail "Expected 'press' to raise exception" + [ $ret -eq $timedout ] && fail "'stbt run' timed out" + [ $ret -ne 0 ] || fail "Expected 'press' to raise exception" } test_that_press_ignores_lircd_broadcast_messages_on_success() { @@ -77,10 +77,10 @@ cat > test.py <<-EOF press("button_that_causes_sighup_and_broadcast_and_timeout") EOF - timeout 10s stbt run -v --control lirc:$lircd_socket:test test.py + timeout 30s stbt run -v --control lirc:$lircd_socket:test test.py ret=$? - [ $? -eq $timedout ] && fail "'press' timed out" - [ $? -ne 0 ] || fail "Expected 'press' to raise exception" + [ $ret -eq $timedout ] && fail "'stbt run' timed out" + [ $ret -ne 0 ] || fail "Expected 'press' to raise exception" } test_press_hold_secs_with_lirc() { @@ -96,11 +96,11 @@ stbt run -v --control lirc:$lircd_socket:test test.py || fail "stbt run failed" - grep -q "fake-lircd: Received: SEND_START test KEY_LEFT" fake-lircd.log && - grep -q "fake-lircd: Received: SEND_STOP test KEY_LEFT" fake-lircd.log || + grep -Eq "fake-lircd: Received: b?'?SEND_START test KEY_LEFT" fake-lircd.log && + grep -Eq "fake-lircd: Received: b?'?SEND_STOP test KEY_LEFT" fake-lircd.log || fail "press: fake-lircd dind't see SEND_START and SEND_STOP" - grep -q "fake-lircd: Received: SEND_START test KEY_RIGHT" fake-lircd.log && - grep -q "fake-lircd: Received: SEND_STOP test KEY_RIGHT" fake-lircd.log || + grep -Eq "fake-lircd: Received: b?'?SEND_START test KEY_RIGHT" fake-lircd.log && + grep -Eq "fake-lircd: Received: b?'?SEND_STOP test KEY_RIGHT" fake-lircd.log || fail "pressing: fake-lircd didn't see SEND_START and SEND_STOP" } diff -Nru stb-tester-30-5-gbefe47c/tests/test_match.py stb-tester-31/tests/test_match.py --- stb-tester-30-5-gbefe47c/tests/test_match.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_match.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,13 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order +from future.utils import string_types + +import os import random +import re import timeit import cv2 @@ -7,6 +16,8 @@ import stbt from _stbt import cv2_compat +from _stbt.imgutils import _image_region +from _stbt.logging import scoped_debug_level from _stbt.match import _merge_regions from tests.test_core import _find_file @@ -30,7 +41,8 @@ def test_that_matchresult_str_image_matches_template_passed_to_match(): - assert "image=\'black.png\'" in str(stbt.match("black.png", frame=black())) + assert re.search(r"image=u?'black.png'", + str(stbt.match("black.png", frame=black()))) def test_that_matchresult_str_image_matches_template_passed_to_match_custom(): @@ -56,6 +68,44 @@ match_parameters=mp(match_method=match_method)) +def test_match_error_message_for_too_small_frame_and_region(): + stbt.match("videotestsrc-redblue.png", frame=black(width=92, height=160)) + stbt.match("videotestsrc-redblue.png", frame=black(), + region=stbt.Region(x=1188, y=560, width=92, height=160)) + + with pytest.raises(ValueError) as excinfo: + stbt.match("videotestsrc-redblue.png", + frame=black(width=91, height=160)) + assert ( + "Frame (160, 91, 3) must be larger than reference image (160, 92, 3)" + in str(excinfo.value)) + + with pytest.raises(ValueError) as excinfo: + stbt.match("videotestsrc-redblue.png", + frame=black(width=92, height=159)) + assert ( + "Frame (159, 92, 3) must be larger than reference image (160, 92, 3)" + in str(excinfo.value)) + + with pytest.raises(ValueError) as excinfo: + # Region seems large enough but actually it extends beyond the frame + stbt.match("videotestsrc-redblue.png", frame=black(), + region=stbt.Region(x=1189, y=560, width=92, height=160)) + assert ( + "Region(x=1189, y=560, right=1280, bottom=720) must be larger than " + "reference image (160, 92, 3)" + in str(excinfo.value)) + + with pytest.raises(ValueError) as excinfo: + # Region seems large enough but actually it extends beyond the frame + stbt.match("videotestsrc-redblue.png", frame=black(), + region=stbt.Region(x=1188, y=561, width=92, height=160)) + assert ( + "Region(x=1188, y=561, right=1280, bottom=720) must be larger than " + "reference image (160, 92, 3)" + in str(excinfo.value)) + + @pytest.mark.parametrize("match_method", [ stbt.MatchMethod.SQDIFF, stbt.MatchMethod.SQDIFF_NORMED, @@ -106,7 +156,7 @@ matches = list(m.region for m in stbt.match_all( 'button.png', frame=stbt.load_image('buttons.png'), match_parameters=mp(match_method=match_method))) - print matches + print(matches) assert plain_buttons == sorted(matches) @@ -120,7 +170,7 @@ 'button.png', frame=frame, match_parameters=mp(match_method=match_method, confirm_method=stbt.ConfirmMethod.NONE))) - print matches + print(matches) assert overlapped_button not in matches assert sorted(plain_buttons + labelled_buttons + [overlapping_button]) == \ sorted(matches) @@ -131,7 +181,7 @@ frame = stbt.load_image("buttons-on-blue-background.png") matches = list(m.region for m in stbt.match_all( "button-transparent.png", frame=frame)) - print matches + print(matches) assert overlapped_button not in matches assert (sorted(plain_buttons + labelled_buttons + [overlapping_button]) == sorted(matches)) @@ -145,6 +195,150 @@ @requires_opencv_3 +def test_transparent_reference_image_with_hard_edge(): + # This is a regression test for a bug in the pyramid optimisation when + # the reference image has a very small number of pixels, or the only non- + # transparent pixels are near the edges of reference image: + # At the smaller pyramid levels, the pixels near the edge of the reference + # image won't match exactly because the corresponding pixels in the + # down-scaled frame have been blurred. We also blur the reference image + # before down-scaling, but since it doesn't know what's outside its edges, + # it won't have the blurring near the edge. + frame = stbt.load_image("images/regression/roku-tile-frame.png") + m = stbt.match("images/regression/roku-tile-selection.png", frame=frame) + assert m + assert stbt.Region(x=325, y=145, right=545, bottom=325).contains(m.region) + + +@pytest.mark.parametrize("frame,image", [ + # pylint:disable=bad-whitespace,line-too-long + ("images/regression/badpyramid-frame.png", "images/regression/badpyramid-reference.png"), + ("images/regression/badpyramid-frame2.png", "images/regression/badpyramid-reference2.png"), +]) +@pytest.mark.parametrize("match_method,match_threshold", [ + (stbt.MatchMethod.SQDIFF, 0.98), + (stbt.MatchMethod.SQDIFF_NORMED, 0.8), + (stbt.MatchMethod.CCORR_NORMED, 0.8), + (stbt.MatchMethod.CCOEFF_NORMED, 0.8), +]) +def test_pyramid_roi_too_small(frame, image, match_method, match_threshold): + # This is a regression test for an error that was seen with a particular + # frame from a single test-run, with SQDIFF_NORMED: + # cv2.error: (-215) _img.size().height <= _templ.size().height && + # _img.size().width <= _templ.size().width in function matchTemplate + with scoped_debug_level(2): + stbt.match( + image, + frame=stbt.load_image(frame), + match_parameters=stbt.MatchParameters( + match_method=match_method, + match_threshold=match_threshold)) + + +@requires_opencv_3 +@pytest.mark.parametrize("image,expected", [ + # pylint:disable=bad-whitespace,line-too-long + ("red-blue-columns", stbt.Region(x=0, y=0, width=40, height=40)), + ("red-blue-columns-transparent", stbt.Region(x=0, y=0, width=40, height=40)), + ("blue-red-columns", stbt.Region(x=1240, y=680, width=40, height=40)), + ("blue-red-columns-transparent", stbt.Region(x=1240, y=680, width=40, height=40)), + ("red-blue-rows", stbt.Region(x=1240, y=0, width=40, height=40)), + ("red-blue-rows-transparent", stbt.Region(x=1240, y=0, width=40, height=40)), + ("blue-red-rows", stbt.Region(x=0, y=680, width=40, height=40)), + ("blue-red-rows-transparent", stbt.Region(x=0, y=680, width=40, height=40)), + ("red-dots", stbt.Region(x=280, y=302, width=21, height=21)), + ("red-dots-1px-border", stbt.Region(x=279, y=301, width=23, height=23)), + ("blue-dots", stbt.Region(x=307, y=303, width=21, height=21)), + ("blue-dots-1px-border", stbt.Region(x=306, y=302, width=23, height=23)), +]) +def test_match_region(image, expected): + frame = stbt.load_image("images/region/frame.png") + m = stbt.match("images/region/%s.png" % image, frame=frame) + assert m + assert m.region == expected + + +@pytest.mark.parametrize("x,y", [ + # "-20" means "1 image width away from the frame's right or bottom edge". + (0, 0), (-20, 0), (0, -20), (-20, -20), # corners + (50, 0), (-20, 50), (50, -20), (0, 50), # edges +]) +@pytest.mark.parametrize("offset_x,offset_y", [ + # Right at the edge or corner; 1px away horizontally, or vertically, or + # both; 2px away etc. + (0, 0), + (1, 0), + (0, 1), + (1, 1), + (2, 0), + (0, 2), + (2, 2), + (3, 3), + (4, 4), + (5, 5), + # These are outside of the frame so they shouldn't match + (-1, 0), + (0, -1), + (-1, -1), + (-2, 0), + (0, -2), + (-2, -2), + (-10, 0), + (0, -10), + (-10, -10), +]) +@pytest.mark.parametrize("width,height", [ + (20, 20), + (20, 21), + (21, 20), + (21, 21), + (31, 30), + (40, 40), + (40, 41), + (41, 40), + (41, 41), + (158, 88), +]) +@pytest.mark.parametrize("frame_width,frame_height", [ + (160, 90), + (159, 89), # an odd-sized "frame" can happen if the user gives a region +]) +def test_match_region2(x, y, offset_x, offset_y, width, height, + frame_width, frame_height): + if x >= 0: + x = x + offset_x + else: + x = frame_width - width - offset_x + if y >= 0: + y = y + offset_y + else: + y = frame_height - height - offset_y + region = stbt.Region(x, y, width=width, height=height) + image = numpy.ones((region.height, region.width, 3), numpy.uint8) * 255 + image[5:-5, 5:-5] = (255, 0, 0) + frame = black(frame_width, frame_height) + frame[region.to_slice()] = image[ + 0 if region.y >= 0 else -region.y : + region.height if region.bottom <= frame_height + else frame_height - region.y, + 0 if region.x >= 0 else -region.x : + region.width if region.right <= frame_width + else frame_width - region.x] * .85 + # N.B. .85 is the lowest at which all the tests still passed when I + # disabled the pyramid optimisation. + + should_match = _image_region(frame).contains(region) + + m = stbt.match(image, frame) + if should_match != m.match or os.environ.get("STBT_DEBUG"): + with scoped_debug_level(2): + stbt.match(image, frame) + assert should_match == m.match + if should_match: + assert m.region == region + + +@requires_opencv_3 def test_that_match_all_can_be_used_with_ocr_to_read_buttons(): # Demonstrates how match_all can be used with ocr for UIs consisting of text # on buttons @@ -156,7 +350,7 @@ m.region.extend(x=30, y=10, right=-30, bottom=-10))) for m in stbt.match_all('button-transparent.png', frame=frame)] text = sorted([t for t in text if t not in ['', '\\s']]) - print text + print(text) assert text == [u'Button 1', u'Button 2', u'Buttons'] @@ -171,7 +365,7 @@ all_matches = set() for m in stbt.match_all("action-panel-template.png", frame=frame, match_parameters=mp(match_method=match_method)): - print m + print(m) assert m.region not in all_matches, "Match %s already seen:\n %s" % ( m, "\n ".join(str(x) for x in all_matches)) assert all(stbt.Region.intersect(m.region, x) is None @@ -195,7 +389,7 @@ "button.png", frame=stbt.load_image("buttons.png"), match_parameters=mp(match_method=match_method), region=stbt.Region(x=160, y=60, right=340, bottom=190))) - print matches + print(matches) assert matches == [stbt.Region(x, y, width=135, height=44) for x, y in [ (177, 75), (177, 119)]] @@ -214,7 +408,7 @@ for x in range(0, 320, 16) for y in range(0, 240, 16)]) - print matches + print(matches) assert matches == expected_matches @@ -247,9 +441,9 @@ from _stbt.match import _build_pyramid mask = numpy.ones((20, 20, 3), dtype=numpy.uint8) * 255 - mask[3:9, 3:9] = 0 # first 0 is an even row/col, last 0 is an odd row/col + mask[5:9, 5:9] = 0 # first 0 is an even row/col, last 0 is an odd row/col n = mask.size - numpy.count_nonzero(mask) - assert n == 6 * 6 * 3 + assert n == 4 * 4 * 3 cv2.imwrite("/tmp/dave1.png", mask) mask_pyramid = _build_pyramid(mask, 2, is_mask=True) @@ -257,22 +451,18 @@ downsampled = mask_pyramid[1] cv2.imwrite("/tmp/dave2.png", downsampled) - assert downsampled.shape == (10, 10, 3) - print downsampled[:, :, 0] # pylint:disable=unsubscriptable-object - n = downsampled.size - numpy.count_nonzero(downsampled) - assert 3 * 3 * 3 <= n <= 6 * 6 * 3 + assert downsampled.shape == (8, 8, 3) + print(downsampled[:, :, 0]) # pylint:disable=unsubscriptable-object expected = [ # pylint:disable=bad-whitespace - [255, 255, 255, 255, 255, 255, 255, 255, 255, 255], - [255, 0, 0, 0, 0, 0, 255, 255, 255, 255], - [255, 0, 0, 0, 0, 0, 255, 255, 255, 255], - [255, 0, 0, 0, 0, 0, 255, 255, 255, 255], - [255, 0, 0, 0, 0, 0, 255, 255, 255, 255], - [255, 0, 0, 0, 0, 0, 255, 255, 255, 255], - [255, 255, 255, 255, 255, 255, 255, 255, 255, 255], - [255, 255, 255, 255, 255, 255, 255, 255, 255, 255], - [255, 255, 255, 255, 255, 255, 255, 255, 255, 255], - [255, 255, 255, 255, 255, 255, 255, 255, 255, 255]] + [255, 255, 255, 255, 255, 255, 255, 255], + [255, 0, 0, 0, 0, 255, 255, 255], + [255, 0, 0, 0, 0, 255, 255, 255], + [255, 0, 0, 0, 0, 255, 255, 255], + [255, 0, 0, 0, 0, 255, 255, 255], + [255, 255, 255, 255, 255, 255, 255, 255], + [255, 255, 255, 255, 255, 255, 255, 255], + [255, 255, 255, 255, 255, 255, 255, 255]] assert numpy.all(downsampled[:, :, 0] == expected) # pylint:disable=unsubscriptable-object @@ -317,7 +507,7 @@ ("button-transparent.png", "buttons.png"), ] for reference, frame in images: - if isinstance(frame, (str, unicode)): + if isinstance(frame, string_types): frame = stbt.load_image(frame, cv2.IMREAD_COLOR) reference = _load_image(reference) orig_m = stbt.match(reference, frame=frame) @@ -367,6 +557,6 @@ times = timeit.repeat(lambda: _merge_regions(regions[:]), number=1, repeat=10) - print times - print min(times) + print(times) + print(min(times)) assert min(times) < (0.001 * n / 20) diff -Nru stb-tester-30-5-gbefe47c/tests/test-match.sh stb-tester-31/tests/test-match.sh --- stb-tester-30-5-gbefe47c/tests/test-match.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-match.sh 2019-09-18 14:04:32.000000000 +0000 @@ -2,7 +2,7 @@ skip_if_opencv_2() { local version=$( - python -c 'from _stbt.cv2_compat import version; print version[0]') + $python -c 'from _stbt.cv2_compat import version; print(version[0])') [[ "$version" -ge 3 ]] || skip "Skipping because OpenCV version < 3" } @@ -211,8 +211,8 @@ EOF ! stbt run -vv test.py \ || fail "Trying to match an non-existant template should throw" - cat log | grep -q "FAIL: test.py: IOError: No such file: idontexist.png" \ - || fail "Didn't see 'IOError: No such file'" + cat log | grep -Eq "FAIL: test.py: (IO|OS)Error: No such file: idontexist.png" \ + || fail "Didn't see 'No such file'" } test_match_invalid_template() { @@ -397,7 +397,7 @@ for search_area in [Region.ALL, Region(228, 0, 92, 160), Region(200, 0, 300, 400), Region(200, 0, 300, 400), Region(-200, -100, 600, 800)]: - print "\nSearch Area:", search_area + print("\nSearch Area: %s" % (search_area,)) match_result = match("$testdir/videotestsrc-redblue.png", region=search_area) assert match_result and match_result.region == Region(228, 0, 92, 160) @@ -407,7 +407,7 @@ for search_area in [Region(228, 3, 92, 260), Region(10, 0, 300, 200), Region(-210, -23, 400, 200)]: - print "Search Area:", search_area + print("Search Area: %s" % (search_area,)) assert not match("$testdir/videotestsrc-redblue.png", region=search_area) try: diff -Nru stb-tester-30-5-gbefe47c/tests/test_motion.py stb-tester-31/tests/test_motion.py --- stb-tester-30-5-gbefe47c/tests/test_motion.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_motion.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import time from contextlib import contextmanager @@ -23,7 +28,7 @@ with MockTime().patch(): res = stbt.wait_for_motion( consecutive_frames='2/4', frames=fake_frames()) - print res + print(res) assert res.time == 1466084606. @@ -31,7 +36,7 @@ with MockTime().patch(): res = stbt.wait_for_motion( consecutive_frames='2/3', frames=fake_frames()) - print res + print(res) assert res.time == 1466084606. @@ -57,6 +62,11 @@ stbt.wait_for_motion(consecutive_frames=2, frames=fake_frames()) +def test_that_wait_for_motion_detects_a_wipe(): + stbt.wait_for_motion(consecutive_frames="10/30", frames=wipe()) + stbt.wait_for_motion(frames=gradient_wipe()) + + def fake_frames(): a = numpy.zeros((2, 2, 3), dtype=numpy.uint8) a.flags.writeable = False @@ -76,6 +86,54 @@ yield stbt.Frame(x, time=t) +def wipe(): + frame = numpy.zeros((720, 1280, 3), dtype=numpy.uint8) + for x in range(0, 720, 2): + frame[x:x + 2, :, :] = 255 + yield stbt.Frame(frame, time=x / 30.) + + +def clamp(x, bottom, top): + return min(top, max(bottom, x)) + + +def gradient_wipe(min_=100, max_=200, swipe_height=40): + """Use write_video(gradient_wipe()) to see what this looks like.""" + frame = min_ * numpy.ones( + (720 + swipe_height * 4, 1280, 3), dtype=numpy.uint8) + diff = max_ - min_ + + # detect_motion ignores differences of under 40, so what's the fastest we + # can wipe while making sure the inter-frame differences are always under + # 40?: + speed = 40 * swipe_height / diff + + print("pixel difference: %f" % (diff / swipe_height)) + print("max_speed: %f" % speed) + + edge = numpy.ones((swipe_height * 3, 1280, 3), dtype=numpy.uint8) * min_ + for n in range(swipe_height * 3): + edge[n, :, :] = clamp(max_ - (n - swipe_height) * diff / swipe_height, + min_, max_) + + for x in range(0, frame.shape[0] - swipe_height * 3, int(speed)): + frame[x:x + swipe_height * 3, :, :] = edge + yield stbt.Frame(frame[swipe_height * 2:swipe_height * 2 + 720], + time=x / 30.) + + +def write_video(g): + """This was useful during the development of wipe and gradient_wipe. + Usage: write_video(gradient_wipe())""" + import cv2 + + vw = cv2.VideoWriter("test.avi", cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), + 30, (1280, 720)) + for frame in g: + vw.write(frame) + vw.release() + + class MockTime(object): def __init__(self, start_time=1466084600.): self._time = start_time @@ -97,8 +155,8 @@ raise exception self.at(0, raise_exception) - def at(self, offset, function): - self._functions.append((self._time + offset, function)) + def at(self, offset, func): + self._functions.append((self._time + offset, func)) self._functions.sort() @contextmanager diff -Nru stb-tester-30-5-gbefe47c/tests/test-motion.sh stb-tester-31/tests/test-motion.sh --- stb-tester-30-5-gbefe47c/tests/test-motion.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-motion.sh 2019-09-18 14:04:32.000000000 +0000 @@ -49,9 +49,7 @@ press("OK") wait_for_motion(mask="idontexist.png") EOF - timeout 10 stbt run -v test.py &> test.log - local ret=$? - [ $ret -ne $timedout -a $ret -ne 0 ] || fail "Unexpected exit status $ret" + ! stbt run -v test.py &> test.log || fail "Expected script to fail" grep -q "No such file: idontexist.png" test.log || fail "Expected 'No such file: idontexist.png' but saw '$( grep 'No such file' test.log | head -n1)'" @@ -249,6 +247,7 @@ --source-pipeline "multifilesrc location=$testdir/box-%05d.png loop=true \ ! image/png,framerate=25/1" \ --sink-pipeline 'gdppay ! filesink location=fifo' \ + --save-screenshot=never \ detect_motion.py & source_pid=$! trap "kill $source_pid; rm fifo" EXIT @@ -260,6 +259,28 @@ --source-pipeline 'filesrc location=fifo ! gdpdepay' \ verify.py } + +test_press_and_wait_visualisation() { + cat > press_and_wait.py <<-EOF && + import stbt + stbt.press_and_wait("ball") + EOF + mkfifo fifo || fail "Initial test setup failed" + + stbt run -v \ + --sink-pipeline 'gdppay ! filesink location=fifo' \ + --save-screenshot=never \ + press_and_wait.py & + source_pid=$! + trap "kill $source_pid; rm fifo" EXIT + + cat > verify.py <<-EOF && + wait_for_match("$testdir/press_and_wait_visualisation.png") + EOF + stbt run -v --control=none \ + --source-pipeline 'filesrc location=fifo ! gdpdepay' \ + verify.py +} test_that_wait_for_motion_returns_first_frame_with_motion() { # Frames are: (0) black - (1) black - (2) black - (3) green - repeat diff -Nru stb-tester-30-5-gbefe47c/tests/test_ocr.py stb-tester-31/tests/test_ocr.py --- stb-tester-30-5-gbefe47c/tests/test_ocr.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_ocr.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,5 +1,10 @@ # coding: utf-8 +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import re from contextlib import contextmanager @@ -42,7 +47,7 @@ def test_that_ocr_reads_unicode(): text = stbt.ocr(frame=load_image('ocr/unicode.png'), lang='eng+deu') - assert isinstance(text, unicode) + assert isinstance(text, str) assert u'£500\nDavid Röthlisberger' == text @@ -149,10 +154,19 @@ tesseract_user_patterns=patterns) +def test_char_whitelist(): + # Without char_whitelist tesseract reads "OO" (the letter oh). + assert u'00' == stbt.ocr( + frame=load_image('ocr/00.png'), + mode=stbt.OcrMode.SINGLE_WORD, + char_whitelist="0123456789") + + @pytest.mark.parametrize("words", [ pytest.param(None, marks=pytest.mark.xfail), ['192.168.10.1'], - '192.168.10.1', + b'192.168.10.1', + u'192.168.10.1', ]) def test_user_dictionary_with_non_english_language(words): assert u'192.168.10.1' == stbt.ocr( @@ -210,7 +224,7 @@ assert re.match( r"TextMatchResult$time=None, match=True, region=Region\(.*$, " - r"frame=<1280x720x3>, text=u'Onion Bhaji'\)", + r"frame=<1280x720x3>, text=u?'Onion Bhaji'\)", str(result)) diff -Nru stb-tester-30-5-gbefe47c/tests/test-ocr.sh stb-tester-31/tests/test-ocr.sh --- stb-tester-30-5-gbefe47c/tests/test-ocr.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-ocr.sh 2019-09-18 14:04:32.000000000 +0000 @@ -4,7 +4,7 @@ cat > test.py <<-EOF import stbt - stbt.frames(timeout_secs=60).next() # wait 'til video pipeline playing + next(stbt.frames(timeout_secs=60)) # wait 'til video pipeline playing text = stbt.ocr() assert text == "Hello there", "Unexpected text: %s" % text diff -Nru stb-tester-30-5-gbefe47c/tests/test-packaging.sh stb-tester-31/tests/test-packaging.sh --- stb-tester-30-5-gbefe47c/tests/test-packaging.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-packaging.sh 2019-09-18 14:04:32.000000000 +0000 @@ -7,43 +7,44 @@ "$testdir/videotestsrc-redblue.png", frame=cv2.imread("$testdir/videotestsrc-full-frame.png")) EOF - python test.py + $python test.py } test_that_stbt_imports_the_installed_version() { cat > test.py <<-EOF - import re, stbt - print stbt.__file__ - print stbt._stbt.__file__ - def firstdir(path): - return re.match("/?[^/]+", path).group() - assert firstdir(stbt.__file__) == firstdir(stbt._stbt.__file__) + import os, re, stbt + print(stbt.__file__) + print(stbt._stbt.__file__) + print(os.environ["PYTHONPATH"]) + prefix, _ = os.environ["PYTHONPATH"].split(":", 1) + assert stbt.__file__.startswith(prefix) + assert stbt._stbt.__file__.startswith(prefix) EOF - python test.py || fail "Python imported the wrong _stbt" + $python test.py || fail "Python imported the wrong _stbt" stbt run test.py || fail "stbt run imported the wrong _stbt" } test_that_stbt_imports_the_source_version() { - (cd "$srcdir" && python <<-EOF) || fail "Python from srcdir imported the wrong _stbt" - import stbt - print stbt.__file__ - print stbt._stbt.__file__ - assert stbt.__file__.startswith("stbt/__init__.py") - assert stbt._stbt.__file__.startswith("_stbt/__init__.py") + (cd "$srcdir" && $python <<-EOF) || fail "Python from srcdir imported the wrong _stbt" + import re, stbt + print(stbt.__file__) + print(stbt._stbt.__file__) + print("$srcdir/") + assert re.match(r"($srcdir/)?stbt/__init__.pyc?$", stbt.__file__) + assert re.match(r"($srcdir/)?_stbt/__init__.pyc?$", stbt._stbt.__file__) EOF cat > test.py <<-EOF import re, stbt - print stbt.__file__ - print stbt._stbt.__file__ - def firstdir(path): - return re.match("/?[^/]+", path).group() - assert firstdir(stbt.__file__) == firstdir(stbt._stbt.__file__) + print(stbt.__file__) + print(stbt._stbt.__file__) + assert re.match(r"$srcdir/stbt/__init__.pyc?$", stbt.__file__) + assert re.match(r"$srcdir/_stbt/__init__.pyc?$", stbt._stbt.__file__) EOF - PYTHONPATH="$srcdir" python test.py || + PYTHONPATH="$srcdir" $python test.py || fail 'Python with PYTHONPATH=$srcdir imported the wrong _stbt' - "$srcdir"/stbt-run "$scratchdir"/test.py || + $python "$srcdir"/stbt_run.py "$scratchdir"/test.py || fail "stbt run imported the wrong _stbt" } diff -Nru stb-tester-30-5-gbefe47c/tests/test_power.py stb-tester-31/tests/test_power.py --- stb-tester-30-5-gbefe47c/tests/test_power.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_power.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,9 @@ """Tests for the _ATEN_PE6108G PDU class""" +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order from contextlib import contextmanager import pytest diff -Nru stb-tester-30-5-gbefe47c/tests/test_press.py stb-tester-31/tests/test_press.py --- stb-tester-30-5-gbefe47c/tests/test_press.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_press.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import pytest from _stbt.core import DeviceUnderTest, NoSinkPipeline @@ -32,13 +37,13 @@ self.keyup_called = 0 def keydown(self, key): - print "keydown %s" % key + print("keydown %s" % key) self.keydown_called += 1 if self.raises_on_keydown: raise RuntimeError("keydown %s failed" % key) def keyup(self, key): - print "keyup %s" % key + print("keyup %s" % key) self.keyup_called += 1 if self.raises_on_keyup: raise RuntimeError("keyup %s failed" % key) diff -Nru stb-tester-30-5-gbefe47c/tests/test.py stb-tester-31/tests/test.py --- stb-tester-30-5-gbefe47c/tests/test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,24 +0,0 @@ -import glob -import os -import sys -import time - -import stbt - -for arg in sys.argv[1:]: - print "Command-line argument: %s\n" % arg - -# Fail if this script is run more than once from the same $scratchdir -n_runs = len(glob.glob("../????-??-??_??.??.??*")) # includes current run -if n_runs == 2: - raise stbt.UITestError("Not the device-under-test's fault") -elif n_runs > 2: # UITestFailure - stbt.wait_for_match("videotestsrc-checkers-8.png", timeout_secs=1) - -stbt.press("gamut") -stbt.wait_for_match("videotestsrc-gamut.png") - -time.sleep(float(os.getenv("sleep", 0))) - -stbt.press("smpte") -stbt.wait_for_motion() diff -Nru stb-tester-30-5-gbefe47c/tests/test-python-2.sh stb-tester-31/tests/test-python-2.sh --- stb-tester-30-5-gbefe47c/tests/test-python-2.sh 1970-01-01 00:00:00.000000000 +0000 +++ stb-tester-31/tests/test-python-2.sh 2019-09-18 14:04:32.000000000 +0000 @@ -0,0 +1,35 @@ +test_python_2_script() { + [[ "$python_version" == "2.7" ]] || skip "Requires Python 2" + cat > test.py <<-EOF + print "hi" + EOF + stbt run -v test.py +} + +test_python_2_function() { + [[ "$python_version" == "2.7" ]] || skip "Requires Python 2" + cat > test.py <<-EOF + def test_print_statement(): + print "hi" + EOF + stbt run -v test.py::test_print_statement +} + +test_python_2_script_with_python3_compat() { + cat > test.py <<-EOF + from __future__ import print_function + print "hi" + EOF + ! stbt run -v test.py || fail "stbt-run should fail" + cat log | grep SyntaxError || fail "Didn't see 'SyntaxError'" +} + +test_python_2_function_with_python3_compat() { + cat > test.py <<-EOF + from __future__ import print_function + def test_print_statement(): + print "hi" + EOF + ! stbt run -v test.py || fail "stbt-run should fail" + cat log | grep SyntaxError || fail "Didn't see 'SyntaxError'" +} diff -Nru stb-tester-30-5-gbefe47c/tests/test-soak.sh stb-tester-31/tests/test-soak.sh --- stb-tester-30-5-gbefe47c/tests/test-soak.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-soak.sh 2019-09-18 14:04:32.000000000 +0000 @@ -14,27 +14,27 @@ with open("/proc/%s/stat" % os.getpid()) as f: stat = f.read() rss = int(stat.split()[23]) * 4 - print "VmRSS: %s kB" % rss + print("VmRSS: %s kB" % rss) global initial_rss if initial_rss is None: initial_rss = rss return rss - print "Testing short stbt.frames" + print("Testing short stbt.frames") end_time = time.time() + 600 # 10 minutes while time.time() < end_time: for frame in stbt.frames(timeout_secs=10): stbt.match("$testdir/videotestsrc-redblue-flipped.png", frame) assert get_rss() < initial_rss * 1.1 - print "Testing long stbt.frames" + print("Testing long stbt.frames") for frame in stbt.frames(timeout_secs=600): # 10 minutes stbt.match("$testdir/videotestsrc-redblue-flipped.png", frame) if int(frame.time) % 10 == 0: assert get_rss() < initial_rss * 1.1 time.sleep(1) - print "Testing stbt.get_frame" + print("Testing stbt.get_frame") end_time = time.time() + 600 # 10 minutes while time.time() < end_time: frame = stbt.get_frame() diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-auto-selftest.sh stb-tester-31/tests/test-stbt-auto-selftest.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-auto-selftest.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-auto-selftest.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,184 +0,0 @@ -cd_example_testpack() -{ - cp -R "$testdir/auto-selftest-example-test-pack" test-pack && - cd test-pack || fail "Test setup failed" - export STBT_CONFIG_FILE="$PWD/.stbt.conf" -} - -check_generated_files() -{ - diff -ur --exclude="*.pyc" --exclude=__pycache__ \ - "$testdir"/auto-selftest-example-test-pack/selftest/auto_selftest \ - ./selftest/auto_selftest || fail "Unexpected changes to selftest files" -} - -test_auto_selftest_generate() -{ - cd_example_testpack && - stbt auto-selftest generate && - diff -ur --exclude="*.pyc" --exclude=__pycache__ \ - "$testdir"/auto-selftest-example-test-pack . -} - -test_auto_selftest_generate_first_time() { - cd_example_testpack && - rm -rf selftest/auto_selftest && - stbt auto-selftest generate && - diff -ur --exclude="*.pyc" --exclude=__pycache__ \ - "$testdir"/auto-selftest-example-test-pack . -} - -test_that_generated_auto_selftests_pass_as_doctests() -{ - PYTHONPATH=$srcdir python -m doctest \ - "$testdir/auto-selftest-example-test-pack/selftest/auto_selftest/tests/example_selftest.py" -} - -test_that_generated_auto_selftests_pass_stbt_auto_selftest_validate() -{ - cd_example_testpack && - stbt auto-selftest validate -} - -test_that_no_auto_selftests_fail_stbt_auto_selftest_validate() -{ - cd_example_testpack && - rm -r selftest/auto_selftest && - ! stbt auto-selftest validate -} - -test_that_new_auto_selftests_fail_stbt_auto_selftest_validate() -{ - cd_example_testpack && - touch selftest/auto_selftest/new.py && - ! stbt auto-selftest validate -} - -test_that_missing_auto_selftests_fail_stbt_auto_selftest_validate() -{ - cd_example_testpack && - rm selftest/auto_selftest/tests/example_selftest.py && - ! stbt auto-selftest validate -} - -test_that_modified_auto_selftests_fail_stbt_auto_selftest_validate() -{ - cd_example_testpack && - echo "pass" >>selftest/auto_selftest/tests/example_selftest.py && - ! stbt auto-selftest validate -} - -test_that_no_screenshots_passes_stbt_auto_selftest_validate() -{ - cd_example_testpack && - rm -r selftest && - stbt auto-selftest validate -} - -test_that_no_selftests_expressions_passes_stbt_auto_selftest_validate() -{ - cd_example_testpack && - rm -rf tests_in_root.py \ - tests/example.py tests/unicode_example.py \ - tests/subdir/subsubdir/subdir_example.py \ - tests/package \ - selftest/auto_selftest && - stbt auto-selftest validate -} - -test_auto_selftest_caching() -{ - cd_example_testpack && - - export XDG_CACHE_HOME=$PWD/cache - - # Regenerating the GStreamer plugin cache would have messed up our timing so - # force it here: - gst-inspect-1.0 &>/dev/null - - STBT_DISABLE_CACHING=1 /usr/bin/time --format=%e -o without_cache.time \ - stbt auto-selftest validate - - ! [ -e cache/stbt/cache.lmdb ] \ - || fail "Cache file created despite STBT_DISABLE_CACHING" - - /usr/bin/time --format=%e -o cold_cache.time \ - stbt auto-selftest validate \ - || fail "auto-selftest failed" - - [ -e cache/stbt/cache.lmdb ] || fail "Cache file not created" - - /usr/bin/time --format=%e -o hot_cache.time \ - stbt auto-selftest validate \ - || fail "auto-selftest failed" - - tail without_cache.time cold_cache.time hot_cache.time - - python -c "assert ($( stbt.log \ - || { cat stbt.log; return 1; } - - check_generated_files - - grep -q "warning: 'tests/example_with_no_tests.py' doesn't define any selftests" stbt.log \ - || fail "Didn't find expected warning in 'stbt auto-selftest' output" -} - -test_auto_selftest_generate_with_single_empty_source_file_deletes_selftest() { - cd_example_testpack && - echo "" > tests/example.py && - stbt auto-selftest generate tests/example.py || - fail "stbt auto-selftest failed" - - ! [[ -e selftest/auto_selftest/tests/example_selftest.py ]] || - fail "example_selftest.py wasn't deleted" -} - -test_auto_selftest_generate_with_two_source_files() { - cd_example_testpack && - rm selftest/auto_selftest/tests/example_selftest.py \ - selftest/auto_selftest/tests/unicode_example_selftest.py && - stbt auto-selftest generate tests/example.py \ - tests/unicode_example.py && - check_generated_files -} - -test_auto_selftest_generate_with_two_source_files_one_of_which_is_empty() { - cd_example_testpack && - rm selftest/auto_selftest/tests/example_selftest.py && - stbt auto-selftest generate tests/example_with_no_tests.py \ - tests/example.py && - check_generated_files -} diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-batch.sh stb-tester-31/tests/test-stbt-batch.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-batch.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-batch.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,620 +0,0 @@ -# Run with ./run-tests.sh - -create_test_repo() { - which git &>/dev/null || skip "git is not installed" - ( - git init tests && - cd tests && - git config user.name "Stb Tester" && - git config user.email "test-stbt-batch@stb-tester.com" && - cp "$testdir/test.py" "$testdir/test2.py" "$testdir/test_functions.py" \ - "$testdir"/test_{success,error,failure}.py \ - "$testdir/videotestsrc-checkers-8.png" \ - "$testdir/videotestsrc-gamut.png" . && - git add . && - git commit -m "Initial commit" - ) >/dev/null 2>&1 || fail "Failed to set up git repo" -} - -validate_testrun_dir() { - local d="$1" testname="$2" commit="$3" commit_sha="$4" extra_column="$5" - - [[ -f "$d/combined.log" ]] || fail "$d/combined.log not created" - [[ $(cat "$d/exit-status") == 0 ]] || fail "wrong $d/exit-status" - [[ $(cat "$d/test-name") == "$testname" ]] || fail "wrong $d/test-name" - diff -u <(cat "$srcdir"/VERSION) "$d/stbt-version.log" || fail "Wrong $d/stbt-version.log" - [[ -f "$d/video.webm" ]] || fail "$d/video.webm not created" - [[ -f "$d/thumbnail.jpg" ]] || fail "$d/thumbnail.jpg not created" - [[ ! -f "$d/backtrace.log" ]] || fail "$d/backtrace.log shouldn't have been created" - if [[ -n "$commit" ]]; then - [[ $(cat "$d/git-commit") == "$commit" ]] || fail "wrong $d/git-commit" - [[ $(cat "$d/git-commit-sha") == "$expected_commit_sha" ]] \ - || fail "wrong $d/git-commit-sha" - fi - if [[ -n "$extra_column" ]]; then - grep -q "$extra_column" "$d/extra-columns" || fail "extra column not in $d/extra-columns" - fi -} - -validate_html_report() { - local d="$1" testname="$2" commit="$3" commit_sha="$4" extra_column="$5" - local results_root=$(dirname "$d") - - [[ -f "$d/index.html" ]] || fail "$d/index.html not created" - [[ -f "$results_root/index.html" ]] || fail "$results_root/index.html not created" - grep -q "$testname" "$d/index.html" || fail "test name not in $d/index.html" - grep -q "$testname" "$results_root/index.html" || fail "test name not in $results_root/index.html" - if [[ -n "$commit" ]]; then - grep -q "$commit" "$d/index.html" || fail "git commit not in $d/index.html" - grep -q "$commit" "$results_root/index.html" || fail "git commit not in $results_root/index.html" - fi - if [[ -n "$extra_column" ]]; then - grep -q "$extra_column" "$d/index.html" || fail "extra column not in $d/index.html" - grep -q "$extra_column" "$results_root/index.html" || fail "extra column not in $results_root/index.html" - fi -} - -test_stbt_batch_run_once() { - create_test_repo - stbt batch run -1 -t "my label" tests/test.py || - fail "stbt batch run failed" - - local expected_commit="$(git -C tests describe --always)" - local expected_commit_sha="$(git -C tests rev-parse HEAD)" - - validate_testrun_dir "latest-my label" test.py "$expected_commit" "$expected_commit_sha" "my label" - validate_html_report "latest-my label" test.py "$expected_commit" "$expected_commit_sha" "my label" -} - -test_that_stbt_batch_run_will_run_a_specific_function() { - create_test_repo - stbt batch run -o results -1 \ - tests/test_functions.py::test_that_this_test_is_run \ - || fail "Test failed" - [ -e "results/current/touched" ] || fail "Test not run" -} - -test_that_stbt_batch_run_runs_until_failure() { - create_test_repo - timeout 60 stbt batch run tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - - ls -d ????-??-??_??.??.??* > testruns - [[ $(cat testruns | wc -l) -eq 2 ]] || fail "Expected 2 test runs" - first=$(head -1 testruns) - grep -q success $first/failure-reason || fail "Expected 1st testrun to succeed" - grep -q UITestError latest/failure-reason || fail "Expected 2nd testrun to fail with 'UITestError'" - [[ -f $first/thumbnail.jpg ]] || fail "Expected successful testrun to create thumbnail" - [[ ! -f $first/screenshot.png ]] || fail "Expected successful testrun to not create screenshot" - [[ -f latest/thumbnail.jpg ]] || fail "Expected failed testrun to create thumbnail" - [[ -f latest/screenshot.png ]] || fail "Expected failed testrun to create thumbnail" -} - -test_that_stbt_batch_run_continues_after_uninteresting_failure() { - create_test_repo - timeout 120 stbt batch run -k tests/test.py - [[ $? -eq $timedout ]] && fail "'run' timed out" - - ls -d ????-??-??_??.??.??* > testruns - [[ $(cat testruns | wc -l) -eq 3 ]] || fail "Expected 3 test runs" - grep -q success $(head -1 testruns)/failure-reason || - fail "Expected 1st testrun to succeed" - grep -q UITestError $(sed -n 2p testruns)/failure-reason || - fail "Expected 2nd testrun to fail with 'UITestError'" - grep -q MatchTimeout latest/failure-reason || - fail "Expected 3rd testrun to fail with 'MatchTimeout'" -} - -test_stbt_batch_run_when_test_script_isnt_in_git_repo() { - create_test_repo - rm -rf tests/.git - - stbt batch run -1 tests/test.py || fail "stbt batch run failed" - - [[ $(cat latest/exit-status) == 0 ]] || fail "wrong latest/exit-status" - [[ ! -f latest/git-commit ]] || fail "didn't expect to see latest/git-commit" - grep -q tests/test.py latest/test-name || fail "wrong latest/test-name" - grep -q tests/test.py latest/index.html || fail "test name not in latest/index.html" - grep -q tests/test.py index.html || fail "test name not in index.html" -} - -test_signalname() { - sed -n '/^signalname()/,/^}/ p' "$srcdir"/stbt-batch.d/report \ - > signalname.sh && - . signalname.sh && - declare -f signalname || fail "'signalname' not defined" - - sleep 10 & - pid=$! - ( sleep 1; kill $pid ) & - wait $pid - ret=$? - diff -u <(echo sigterm) <(signalname $((ret - 128))) -} - -expect_runner_to_say() { - for i in {1..100}; do - cat log | grep -qF "$1" && return - sleep 0.1 - done - fail "Didn't find '$1' after 10 seconds" -} - -test_stbt_batch_run_sigterm_once() { - create_test_repo - sleep=4 stbt batch run tests/test.py & - runner=$! - expect_runner_to_say "test.py ..." - kill $runner - expect_runner_to_say "waiting for current test to complete" - wait $runner - diff -u <(echo success) latest/failure-reason || fail "Bad failure-reason" -} - -test_stbt_batch_run_sigterm_twice() { - create_test_repo - sleep=10 stbt batch run tests/test.py & - runner=$! - expect_runner_to_say "test.py ..." - kill $runner - expect_runner_to_say "waiting for current test to complete" - kill $runner - expect_runner_to_say "exiting" - wait $runner - diff -u <(echo "killed (sigterm)") latest/failure-reason || - fail "Bad failure-reason" -} - -test_that_stbt_batch_run_passes_arguments_to_script() { - create_test_repo - stbt batch run -1 \ - tests/test.py "a b" c d -- \ - tests/test.py efg hij - - ls -d ????-??-??_??.??.??* > testruns - assert grep 'Command-line argument: a b$' $(head -1 testruns)/stdout.log - assert grep 'Command-line argument: c$' $(head -1 testruns)/stdout.log - assert grep 'Command-line argument: d$' $(head -1 testruns)/stdout.log - assert grep 'Command-line argument: efg$' latest/stdout.log - assert grep 'Command-line argument: hij$' latest/stdout.log - assert grep 'efg' index.html - assert grep 'hij' index.html - assert grep 'efg' latest/index.html - assert grep 'hij' latest/index.html -} - -test_stbt_batch_report_with_symlinks_for_each_testrun() { - # Use case: After you've run `stbt batch run` several times from different - # directories, you gather all results into a single report by symlinking - # each testrun into a single directory. - - create_test_repo - stbt batch run -1 tests/test.py && - mkdir new-report && - ( cd new-report; ln -s ../2* . ) || - fail "report directory structure setup failed" - - stbt batch report --html-only new-report/2* || return - [[ -f new-report/index.html ]] || fail "new-report/index.html not created" -} - -test_stbt_batch_run_with_custom_logging() { - create_test_repo - set_config batch.pre_run "$PWD/my-logger" - set_config batch.post_run "$PWD/my-logger" - - cat > my-logger <<-'EOF' - #!/bin/sh - printf "%s time\t%s\n" $1 "$(date)" >> extra-columns - EOF - chmod u+x my-logger - - stbt batch run -1 tests/test.py - - grep -q 'start time' index.html || - fail "'start time' missing from report" - grep -q 'stop time' index.html || - fail "'stop time' missing from report" -} - -test_stbt_batch_run_with_custom_classifier() { - create_test_repo - set_config batch.classify "$PWD/my-classifier" - - cat > my-classifier <<-'EOF' - #!/bin/bash - if [[ $(cat exit-status) -ne 0 && $(cat test-name) =~ test.py ]]; - then - echo 'Intentional failure' > failure-reason - fi - EOF - chmod u+x my-classifier - - timeout 60 stbt batch run tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - - grep -q 'Intentional failure' index.html || - fail "Custom failure reason missing from report" -} - -test_stbt_batch_run_without_html_reports() { - create_test_repo - set_config batch.classify "$PWD/my-classifier" - cat > my-classifier <<-'EOF' - #!/bin/bash - touch my-classifier-ran - EOF - chmod u+x my-classifier - - timeout 60 stbt batch run -1 --no-html-report tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - - validate_testrun_dir latest test.py - ! [[ -f "latest/index.html" ]] || fail "latest/index.html shouldn't exist" - ! [[ -f index.html ]] || fail "index.html shouldn't exist" - - # classify should still run. - [[ -f latest/my-classifier-ran ]] || fail "Custom classifier didn't run" -} - -test_stbt_batch_run_no_save_video_no_sink_pipeline() { - do_test_stbt_batch_run_no_save_video --no-save-video "" -} - -test_stbt_batch_run_no_save_video_fakesink() { - do_test_stbt_batch_run_no_save_video --no-save-video fakesink -} - -test_stbt_batch_run_no_sink_pipeline() { - do_test_stbt_batch_run_no_save_video "" "" -} - -do_test_stbt_batch_run_no_save_video() { - local no_save_video="$1" sink_pipeline="$2" - - create_test_repo - set_config global.sink_pipeline "$sink_pipeline" - stbt batch run $no_save_video -1 -t "my label" tests/test.py || - fail "stbt batch run failed" - - local expected_commit="$(git -C tests describe --always)" - local expected_commit_sha="$(git -C tests rev-parse HEAD)" - - case "$no_save_video" in - --no-save-video) - ! [ -e "latest-my label/video.webm" ] || - fail "Video was written even though it shouldn't have been";; - *) - [ -e "latest-my label/video.webm" ] || - fail "Video wasn't written even though it should have been";; - esac - - # We still expect an HTML report even if a video is not available - validate_html_report "latest-my label" test.py "$expected_commit" "$expected_commit_sha" "my label" -} - -test_stbt_batch_run_with_custom_recovery_script() { - create_test_repo - set_config batch.recover "$PWD/my-recover" - - cat > my-recover <<-'EOF' - #!/bin/sh - touch powercycle.log - EOF - chmod u+x my-recover - - timeout 60 stbt batch run tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - - grep -q '>powercycle.log' latest/index.html || - fail "Custom recovery script's log missing from report" -} - -test_stbt_batch_run_recovery_exit_status() { - create_test_repo - set_config batch.recover "$PWD/my-recover" - - cat > my-recover <<-'EOF' - #!/bin/sh - exit 1 - EOF - chmod u+x my-recover - - timeout 60 stbt batch run -kk tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - - ls -d ????-??-??_??.??.??* > testruns - [[ $(cat testruns | wc -l) -eq 2 ]] || fail "Expected 2 test runs" -} - -with_retry() { - local count ret - count="$1"; shift - "$@"; ret=$? - if [[ $count -gt 0 && $1 == curl && $ret -eq 7 ]]; # connection refused - then - sleep 1 - echo "Retrying ($count)..." - with_retry $((count - 1)) "$@" - else - return $ret - fi -} - -test_stbt_batch_instaweb() { - wait_for_report() { - local parent=$1 children pid - children=$(ps -o ppid= -o pid= | awk "\$1 == $parent {print \$2}") - for pid in $children; do - if ps -o command= -p $pid | grep -q report; then - echo "Waiting for 'report' (pid $pid)" - # Can't use `wait` because process isn't a direct child of this - # shell. After finishing, `report` becomes a zombie process so - # `ps` shows it as "(bash)" instead of "bash /path/to/report - # ...". - while ps -o command= -p $pid | grep -q report; do - sleep 0.1 - done - echo "'report' completed" - return - fi - wait_for_report $pid - done - } - - create_test_repo - timeout 60 stbt batch run tests/test.py - [[ $? -eq $timedout ]] && fail "'stbt batch run' timed out" - rundir=$(ls -d 20* | tail -1) - assert grep -q UITestError $rundir/failure-reason - assert grep -q UITestError $rundir/index.html - assert grep -q UITestError index.html - - stbt batch instaweb --debug 127.0.0.1:5787 & - server=$! - trap "killtree $server; wait $server" EXIT - expect_runner_to_say 'Running on http://127.0.0.1:5787/' - - with_retry 5 curl --silent --show-error \ - -F 'value=manual failure reason' \ - http://127.0.0.1:5787/$rundir/failure-reason || fail 'Got HTTP failure' - expect_runner_to_say "POST /$rundir/failure-reason" - wait_for_report $server - assert grep -q "manual failure reason" $rundir/failure-reason.manual - assert grep -q "manual failure reason" $rundir/index.html - assert grep -q "manual failure reason" index.html - - curl --silent --show-error \ - -F "value=UITestError: Not the device-under-test's fault" \ - http://127.0.0.1:5787/$rundir/failure-reason || fail 'Got HTTP failure' - expect_runner_to_say "POST /$rundir/failure-reason" - wait_for_report $server - ! [[ -f $rundir/failure-reason.manual ]] || - fail "server didn't delete '$rundir/failure-reason.manual'" - assert ! grep -q "manual failure reason" index.html - assert ! grep -q "manual failure reason" $rundir/index.html - - curl --silent --show-error \ - -F 'value=Hi there £€' \ - http://127.0.0.1:5787/$rundir/notes || fail 'Got HTTP failure' - expect_runner_to_say "POST /$rundir/notes" - wait_for_report $server - assert grep -q 'Hi there £€' $rundir/notes.manual - assert grep -q 'Hi there £€' $rundir/index.html - assert grep -q 'Hi there £€' index.html - - curl --silent --show-error -X POST \ - http://127.0.0.1:5787/$rundir/delete || fail 'Got HTTP failure' - expect_runner_to_say "POST /$rundir/delete" - wait_for_report $server - ! [[ -f $rundir ]] || fail "server didn't hide '$rundir'" - assert ! grep -q 'Hi there £€' index.html -} - -test_that_stbt_batch_instaweb_shows_directory_listing() { - mkdir my-test-session - echo hi > my-test-session/index.html - - stbt batch instaweb --debug 127.0.0.1:5788 & - server=$! - trap "killtree $server; wait $server" EXIT - expect_runner_to_say 'Running on http://127.0.0.1:5788/' - - with_retry 10 curl http://127.0.0.1:5788/ | grep my-test-session || - fail "Didn't find directory listing at '/'" - diff -u my-test-session/index.html \ - <(curl -L http://127.0.0.1:5788/my-test-session) || - fail "Didn't find index.html at '/my-test-session'" - diff -u my-test-session/index.html \ - <(curl http://127.0.0.1:5788/my-test-session/) || - fail "Didn't find index.html at '/my-test-session/'" -} - -test_that_stbt_batch_run_isolates_stdin_of_user_hooks() { - create_test_repo - cat >my-logger <<-EOF - #!/bin/sh - read x - [ -z \$x ] && echo STDIN=None || echo STDIN=\$x - EOF - chmod +x my-logger - - set_config batch.pre_run "$PWD/my-logger" && - - stbt batch run -1 tests/test.py tests/test2.py - cat log | grep -q "STDIN=None" || fail "Data in user script's STDIN" - cat log | grep -q "test2.py ..." || fail "test2.py wasn't executed" -} - -test_that_stbt_batch_run_exits_with_failure_if_any_test_fails() { - create_test_repo - stbt batch run -1 tests/test_success.py || fail "Test should succeed" - cat */combined.log - - ! stbt batch run -1 tests/test_failure.py || fail "Test should fail" - ! stbt batch run -1 tests/test_error.py || fail "Test should fail" - - ! stbt batch run -1 tests/test_success.py tests/test_failure.py \ - || fail "Test should fail" - ! stbt batch run -1 tests/test_failure.py tests/test_success.py \ - || fail "Test should fail" -} - -test_that_stbt_batch_propagates_exit_status_if_running_a_single_test() { - create_test_repo - stbt batch run -1 tests/test_success.py - [ "$?" = 0 ] || fail "Test should succeed" - - stbt batch run -1 tests/test_failure.py - [ "$?" = 1 ] || fail "Test should fail" - - stbt batch run -1 tests/test_error.py - [ "$?" = 2 ] || fail "Test should error" -} - -test_stbt_batch_output_dir() { - create_test_repo - stbt batch run -1 -o "my results" tests/test.py tests/test2.py \ - || fail "Tests should succeed" - - [[ -f "my results"/index.html ]] || fail "'my results/index.html' not created" - ! [[ -f index.html ]] || fail "index.html created in current directory" - grep -q test.py "my results"/*/test-name || fail "First test's results not in 'my results'" - grep -q test2.py "my results"/*/test-name || fail "Second test's results not in 'my results'" - - validate_testrun_dir "my results/latest" test2.py - validate_html_report "my results/latest" test2.py -} - -test_printing_unicode_characters_in_scripts() { - # This testcase documents the current behaviour when printing non-ascii - # byte strings, which isn't necessarily the desired behaviour. - - which unbuffer &>/dev/null || skip "unbuffer is not installed" - - create_test_repo - cat >tests/unicode.py <<-EOF - # coding: utf-8 - import sys - print u" Röthlisberger" - sys.stderr.write(u" Röthlisberger\n") - EOF - - cat >tests/utf8bytestring.py <<-EOF - # coding: utf-8 - import sys - s = u" Röthlisberger\n".encode("utf-8") - print s - sys.stderr.write(s) - EOF - - unset LC_ALL LC_CTYPE LANG - - # We use unbuffer here to provide a tty to `stbt run` to simulate - # interactive use. - - echo "This should fail (non-utf8 capable tty):" - ! LANG=C unbuffer bash -c 'stbt run tests/unicode.py' \ - || fail "stbt run should have failed to write to non-utf8 capable tty" - - echo "To terminal:" && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/unicode.py' && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/utf8bytestring.py' && - - echo "stdout to /dev/null:" && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/unicode.py >/dev/null' && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/utf8bytestring.py >/dev/null' && - - echo "stderr to /dev/null:" && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/unicode.py 2>/dev/null' && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/utf8bytestring.py 2>/dev/null' && - - echo "stdout to file:" && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/unicode.py >mylog1' && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/utf8bytestring.py >mylog2' && - - echo "stderr to file:" && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/unicode.py 2>mylog3' && - LANG=C.UTF-8 unbuffer bash -c 'stbt run tests/utf8bytestring.py 2>mylog4' && - - echo "stbt batch run:" && - stbt batch run -1 tests/unicode.py && - stbt batch run -1 tests/utf8bytestring.py && - - echo "stbt batch run -v:" && - stbt batch run -v -1 tests/unicode.py && - stbt batch run -v -1 tests/utf8bytestring.py -} - -test_that_stbt_batch_run_can_print_exceptions_with_unicode_characters() { - cat > test.py <<-EOF - # coding: utf-8 - assert False, u"ü" - EOF - stbt batch run -1 test.py - cat latest/combined.log latest/failure-reason - grep -E 'FAIL: .*test.py: AssertionError: ü' latest/combined.log || fail - grep 'assert False, u"ü"' latest/combined.log || fail - grep 'AssertionError: ü' latest/failure-reason || fail -} - -test_that_stbt_batch_run_can_print_exceptions_with_encoded_utf8_string() { - cat > test.py <<-EOF - # coding: utf-8 - assert False, u"ü".encode("utf-8") - EOF - stbt batch run -1 test.py - cat latest/combined.log latest/failure-reason - grep -E 'FAIL: .*test.py: AssertionError: ü' latest/combined.log || fail - grep 'assert False, u"ü"' latest/combined.log || fail - grep 'AssertionError: ü' latest/failure-reason || fail -} - -test_that_tests_reading_from_stdin_dont_mess_up_batch_run_test_list() { - cat >test.py <<-EOF - import sys - in_text = sys.stdin.read() - assert in_text == "", "Stdin said \"%s\"" % in_text - EOF - - stbt batch run -1 test.py test.py \ - || fail "Expected test to pass" - - ls -d ????-??-??_??.??.??* > testruns - [[ $(cat testruns | wc -l) -eq 2 ]] || fail "Expected 2 test runs" -} - -test_that_stbt_batch_failure_reason_shows_the_failing_assert_statement() { - create_test_repo - stbt batch run -1 tests/test_functions.py::test_that_asserts_the_impossible - assert grep -q "AssertionError: assert 1 + 1 == 3" latest/failure-reason -} - -test_that_stbt_batch_thumbnail_is_present_even_with_chdir() { - create_test_repo - stbt batch run -1 tests/test_functions.py::test_that_chdirs - assert [ -e latest/thumbnail.jpg ] -} - -test_that_stbt_batch_run_shuffle_runs_tests() { - create_test_repo - stbt batch run -1 --shuffle \ - tests/test_functions.py::test_that_does_nothing \ - tests/test_functions.py::test_that_this_test_is_run - ls -d ????-??-??_??.??.??* > testruns - [[ $(cat testruns | wc -l) -eq 2 ]] || fail "Expected 2 test runs" -} - -test_that_stbt_batch_run_extracts_backtrace_from_core_file() { - grep -q '^core$' /proc/sys/kernel/core_pattern || - echo core | sudo --non-interactive tee /proc/sys/kernel/core_pattern || - skip "Can't set core pattern" - - ulimit -c unlimited - create_test_repo - stbt batch run -1 tests/test_functions.py::test_that_dumps_core - assert [ -e latest/core ] - assert [ -e latest/backtrace.log ] - cat latest/backtrace.log - assert grep -q abort latest/backtrace.log -} diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-completion.sh stb-tester-31/tests/test-stbt-completion.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-completion.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-completion.sh 2019-09-18 14:04:32.000000000 +0000 @@ -79,7 +79,33 @@ } test_completion_filename_possibly_with_test_functions() { - cd "$srcdir" && . stbt-completion + . "$srcdir"/stbt-completion + + mkdir tests + touch tests/test_success.py + cat > tests/test_functions.py <<-EOF + import os + + + def test_that_this_test_is_run(): + open("touched", "w").close() + + + def test_that_does_nothing(): + pass + + + def test_that_asserts_the_impossible(): + assert 1 + 1 == 3 + + + def test_that_chdirs(): + os.chdir("/tmp") + + + def test_that_dumps_core(): + os.abort() + EOF diff - <(_stbt_cur="tests/test_succ" \ _stbt_filename_possibly_with_test_functions) <<-EOF || @@ -120,10 +146,12 @@ } test_completion_filenames() { - cd "$srcdir" && . stbt-completion - diff -u - <(_stbt_filenames "stbt-ca") <<-EOF || - stbt-camera - stbt-camera.d/ + . "$srcdir/stbt-completion" + mkdir a-dir + touch a-file a-dir/one a-dir/two + diff -u - <(_stbt_filenames "a-") <<-EOF || + a-file + a-dir/ EOF fail "unexpected completions for files + directories" } diff -Nru stb-tester-30-5-gbefe47c/tests/test_stbt_control_relay.py stb-tester-31/tests/test_stbt_control_relay.py --- stb-tester-30-5-gbefe47c/tests/test_stbt_control_relay.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_stbt_control_relay.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,6 +1,12 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import os import socket import subprocess +import sys from contextlib import contextmanager from tempfile import NamedTemporaryFile from textwrap import dedent @@ -9,7 +15,7 @@ from _stbt.control import uri_to_control from _stbt.core import wait_until -from _stbt.utils import named_temporary_directory +from _stbt.utils import named_temporary_directory, scoped_process # For py.test fixtures: @@ -17,16 +23,6 @@ @contextmanager -def scoped_process(process): - try: - yield process - finally: - if process.poll() is None: - process.kill() - process.wait() - - -@contextmanager def scoped_path_addition(path): os.environ['PATH'] = "%s:%s" % (path, os.environ['PATH']) try: @@ -99,20 +95,24 @@ os.environ['LISTEN_PID'] = str(os.getpid()) if fd != 3: os.dup2(fd, 3) + if sys.version_info.major > 2: + os.set_inheritable(3, True) # pylint:disable=no-member os.closerange(4, 100) return preexec_fn def test_stbt_control_relay_with_socket_passing(stbt_control_relay_on_path): # pylint: disable=unused-argument - with NamedTemporaryFile(prefix="stbt-control-relay-test-") as tmpfile: + with NamedTemporaryFile(mode="w+", + prefix="stbt-control-relay-test-") as tmpfile: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(('127.0.0.1', 0)) s.listen(5) proc = subprocess.Popen( ["stbt-control-relay", "-vv", "file:" + tmpfile.name], - preexec_fn=socket_passing_setup(s)) + preexec_fn=socket_passing_setup(s), + close_fds=False) with scoped_process(proc): testcontrol = uri_to_control("lirc:%s:%i:stbt" % s.getsockname()) diff -Nru stb-tester-30-5-gbefe47c/tests/test_stbt_debug.py stb-tester-31/tests/test_stbt_debug.py --- stb-tester-30-5-gbefe47c/tests/test_stbt_debug.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_stbt_debug.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import itertools import os import subprocess @@ -19,20 +24,20 @@ matches = list(stbt.match_all( "videotestsrc-redblue-flipped.png", frame=stbt.load_image("videotestsrc-full-frame.png"))) - print matches + print(matches) assert len(matches) == 0 # Multiple matches; first pass stops with a non-match: matches = list(stbt.match_all( "button.png", frame=stbt.load_image("buttons.png"), match_parameters=mp(match_threshold=0.995))) - print matches + print(matches) assert len(matches) == 6 # Multiple matches; second pass stops with a non-match: matches = list(stbt.match_all( "button.png", frame=stbt.load_image("buttons.png"))) - print matches + print(matches) assert len(matches) == 6 # With absdiff: @@ -40,10 +45,11 @@ "button.png", frame=stbt.load_image("buttons.png"), match_parameters=mp(confirm_method="absdiff", confirm_threshold=0.84))) - print matches + print(matches) assert len(matches) == 6 - files = subprocess.check_output("find stbt-debug | sort", shell=True) + files = subprocess.check_output("find stbt-debug | sort", shell=True) \ + .decode("utf-8") assert files == dedent("""\ stbt-debug stbt-debug/00001 @@ -341,7 +347,8 @@ region=stbt.Region(0, 0, 320, 400)): pass - files = subprocess.check_output("find stbt-debug | sort", shell=True) + files = subprocess.check_output("find stbt-debug | sort", shell=True) \ + .decode("utf-8") assert files == dedent("""\ stbt-debug stbt-debug/00001 @@ -423,7 +430,8 @@ stbt.match_text("Summary", f, region=r, text_color=c) stbt.match_text("Summary", f, region=nonoverlapping) - files = subprocess.check_output("find stbt-debug | sort", shell=True) + files = subprocess.check_output("find stbt-debug | sort", shell=True) \ + .decode("utf-8") assert files == dedent("""\ stbt-debug stbt-debug/00001 @@ -487,7 +495,8 @@ stbt.is_screen_black(f, mask="videotestsrc-mask-no-video.png") stbt.is_screen_black(f, region=stbt.Region(0, 0, 160, 120)) - files = subprocess.check_output("find stbt-debug | sort", shell=True) + files = subprocess.check_output("find stbt-debug | sort", shell=True) \ + .decode("utf-8") assert files == dedent("""\ stbt-debug stbt-debug/00001 diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-lint.sh stb-tester-31/tests/test-stbt-lint.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-lint.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-lint.sh 2019-09-18 14:04:32.000000000 +0000 @@ -78,8 +78,8 @@ test_that_stbt_lint_ignores_image_urls() { cat > test.py <<-EOF && - import urllib2 - urllib2.urlopen("http://example.com/image.png") + def urlopen(x): pass + urlopen("http://example.com/image.png") EOF stbt lint --errors-only test.py } @@ -195,17 +195,17 @@ EOF } -test_that_stbt_lint_checks_frame_parameter_in_frameobject_methods() { +test_that_stbt_lint_checks_frameobjects() { cat > test.py <<-EOF - from stbt import FrameObject, match, match_text, ocr, is_screen_black + import stbt def find_boxes(frame=None): pass - class Button(FrameObject): + class Button(stbt.FrameObject): pass - class ModalDialog(FrameObject): + class ModalDialog(stbt.FrameObject): @property def is_visible(self): return find_boxes() and Button() @@ -214,45 +214,74 @@ @property def is_visible(self): return bool( - match("videotestsrc-redblue.png") and - match_text("Error") and - not is_screen_black()) + stbt.match("videotestsrc-redblue.png") and + stbt.match_text("Error") and + not stbt.is_screen_black()) @property def text(self): - return ocr() + return stbt.ocr() + + @property + def another(self): + assert stbt.press_and_wait("KEY_RIGHT") + stbt.press("KEY_OK") + f = stbt.get_frame() + m = stbt.match("videotestsrc-redblue.png", frame=f) + return (m and + stbt.match_text("Error", + frame=stbt.get_frame()) and + stbt.match_text("Error", + stbt.get_frame())) - class Good(FrameObject): + class Good(stbt.FrameObject): @property def is_visible(self): return find_boxes(self._frame) and Button(self._frame) @property def property1(self): - return bool(match("videotestsrc-redblue.png", self._frame)) + return bool(stbt.match("videotestsrc-redblue.png", self._frame)) @property def property2(self): - return bool(match("videotestsrc-redblue.png", frame=self._frame)) + return bool(stbt.match("videotestsrc-redblue.png", + frame=self._frame)) def not_a_property(self): - return bool(match("videotestsrc-redblue.png")) + if not bool(stbt.match("videotestsrc-redblue.png")): + stbt.press("KEY_OK") + return stbt.wait_until( + lambda: stbt.match("videotestsrc-redblue.png")) def normal_test(): - assert match("videotestsrc-redblue.png") + assert stbt.match("videotestsrc-redblue.png") EOF cp "$testdir/videotestsrc-redblue.png" . stbt lint --errors-only test.py > lint.log - assert_lint_log <<-'EOF' + cat > expected.log <<-'EOF' ************* Module test E: 12,15: "find_boxes()" missing "frame" argument (stbt-frame-object-missing-frame) E: 12,32: "Button()" missing "frame" argument (stbt-frame-object-missing-frame) - E: 18,12: "match('videotestsrc-redblue.png')" missing "frame" argument (stbt-frame-object-missing-frame) - E: 19,12: "match_text('Error')" missing "frame" argument (stbt-frame-object-missing-frame) - E: 20,16: "is_screen_black()" missing "frame" argument (stbt-frame-object-missing-frame) - E: 24,15: "ocr()" missing "frame" argument (stbt-frame-object-missing-frame) - EOF + E: 18,12: "stbt.match('videotestsrc-redblue.png')" missing "frame" argument (stbt-frame-object-missing-frame) + E: 19,12: "stbt.match_text('Error')" missing "frame" argument (stbt-frame-object-missing-frame) + E: 20,16: "stbt.is_screen_black()" missing "frame" argument (stbt-frame-object-missing-frame) + E: 24,15: "stbt.ocr()" missing "frame" argument (stbt-frame-object-missing-frame) + E: 28,15: FrameObject properties must not use "stbt.press_and_wait" (stbt-frame-object-property-press) + E: 29, 8: FrameObject properties must not use "stbt.press" (stbt-frame-object-property-press) + E: 30,12: FrameObject properties must use "self._frame", not "get_frame()" (stbt-frame-object-get-frame) + E: 34,38: FrameObject properties must use "self._frame", not "get_frame()" (stbt-frame-object-get-frame) + E: 36,32: FrameObject properties must use "self._frame", not "get_frame()" (stbt-frame-object-get-frame) + EOF + assert_lint_log < expected.log + + # Also test `match` instead of `stbt.match` (etc). + sed -e 's/^import stbt$/from stbt import FrameObject, get_frame, is_screen_black, match, match_text, ocr, press, press_and_wait, wait_until/' \ + -e 's/stbt\.//g' \ + -i test.py expected.log + stbt lint --errors-only test.py > lint.log + assert_lint_log < expected.log } test_that_stbt_lint_ignores_astroid_inference_exceptions() { @@ -268,3 +297,41 @@ E: 2,23: Undefined variable 'InfoPage' (undefined-variable) EOF } + +test_that_stbt_lint_warns_on_assert_true() { + cat > test.py <<-EOF + assert True + assert True, "My message" + EOF + stbt lint --errors-only test.py > lint.log + + assert_lint_log <<-'EOF' + ************* Module test + E: 1, 0: "assert True" has no effect (stbt-assert-true) + E: 2, 0: "assert True" has no effect (stbt-assert-true) + EOF +} + +test_that_stbt_lint_understands_assert_false() { + cat > test.py <<-EOF + def moo(): + assert False + print("Hi there") + + + def cow(a): + if a: + return 5 + # This checks that we've fixed pylint's false positive warning + # about inconsistent-return-statements ("Either all return + # statements in a function should return an expression, or none of + # them should"). + assert False, "My Message" + EOF + stbt lint --disable=missing-docstring,invalid-name --score=no test.py > lint.log + + assert_lint_log <<-'EOF' + ************* Module test + W: 3, 4: Unreachable code (unreachable) + EOF +} diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-py.sh stb-tester-31/tests/test-stbt-py.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-py.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-py.sh 2019-09-18 14:04:32.000000000 +0000 @@ -67,8 +67,8 @@ test_that_frames_returns_at_least_one_frame() { cat > test.py <<-EOF import stbt - stbt.frames(timeout_secs=0).next() - stbt.frames(timeout_secs=0).next() + next(stbt.frames(timeout_secs=0)) + next(stbt.frames(timeout_secs=0)) EOF stbt run -v test.py } @@ -115,13 +115,13 @@ frames = stbt.frames(timeout_secs=10) for frame in frames: black = stbt.is_screen_black(frame) - print "%s: %s" % (frame.time, black) + print("%s: %s" % (frame.time, black)) if black: break assert black, "Failed to find black screen" for frame in frames: black = stbt.is_screen_black(frame) - print "%s: %s" % (frame.time, black) + print("%s: %s" % (frame.time, black)) if not black: break assert not black, "Failed to find non-black screen" @@ -133,17 +133,17 @@ cat > test.py <<-EOF && import stbt for frame in stbt.frames(): - print frame.time + print(frame.time) break for frame in stbt.frames(): - print frame.time + print(frame.time) break frames = stbt.frames() - frame1 = frames.next() + frame1 = next(frames) frames = stbt.frames() # Drop reference to old 'frames'; should be GCd. - frame2 = frames.next() + frame2 = next(frames) frames3 = stbt.frames() - frame3 = frames3.next() # old 'frames' still holds lock + frame3 = next(frames3) # old 'frames' still holds lock EOF timeout 10 stbt run -v test.py && @@ -176,6 +176,8 @@ } test_that_video_index_is_written_on_eos() { + which python2 || skip "Requires Python 2" + _test_that_video_index_is_written_on_eos 5 && return echo "Failed with 5s video; trying again with 20s video" _test_that_video_index_is_written_on_eos 20 @@ -189,7 +191,8 @@ --sink-pipeline \ "queue ! vp8enc cpu-used=6 ! webmmux ! filesink location=video.webm" \ test.py && - "$testdir"/webminspector/webminspector.py video.webm &> webminspector.log && + python2 "$testdir"/webminspector/webminspector.py video.webm \ + &> webminspector.log && grep "Cue Point" webminspector.log || { cat webminspector.log echo "error: Didn't find 'Cue Point' in $scratchdir/webminspector.log" @@ -227,23 +230,6 @@ cat log | grep "verbose: 1" } -test_that_restart_source_option_is_read() { - cat > test.py <<-EOF && - import stbt - print "value: %s" % stbt._dut._display.restart_source_enabled - EOF - # Read from the command line - stbt run -v --restart-source --control none test.py && - cat log | grep "restart_source: True" && - cat log | grep "value: True" && - echo > log && - # Read from the config file - set_config global.restart_source "True" && - stbt run -v --control none test.py && - cat log | grep "restart_source: True" && - cat log | grep "value: True" -} - test_press_visualisation() { cat > press.py <<-EOF && import signal, time @@ -280,7 +266,7 @@ } test_clock_visualisation() { - PYTHONPATH="$srcdir" python -c "import stbt, _stbt.ocr, distutils, sys; \ + PYTHONPATH="$srcdir" $python -c "import stbt, _stbt.ocr, distutils, sys; \ sys.exit(0 if (_stbt.ocr._tesseract_version() \ >= distutils.version.LooseVersion('3.03')) else 77)" case $? in @@ -398,9 +384,9 @@ f = stbt.get_frame() t = time.time() - print "stbt.get_frame().time:", f.time - print "time.time():", t - print "latency:", t - f.time + print("stbt.get_frame().time: %.6f" % f.time) + print("time.time(): %.6f" % t) + print("latency: %.6f" % (t - f.time)) # get_frame() gives us the last frame that arrived. This may arrived a # little time ago and have been waiting in a buffer. @@ -413,7 +399,7 @@ test_template_annotation_labels() { cat > test.py <<-EOF && import stbt - for frame in stbt.frames(timeout_secs=10): + for frame in stbt.frames(timeout_secs=20): stbt.match("${testdir}/videotestsrc-ball.png", frame) EOF mkfifo fifo || fail "Initial test setup failed" @@ -438,28 +424,30 @@ test_template_annotation_with_ndarray_template() { cat > test.py <<-EOF && - import stbt, numpy as np - template = np.ones(shape=(100, 100, 3), dtype=np.uint8) - template *= np.array([0, 255, 0], dtype=np.uint8) # green - stbt.save_frame(template, 'template.png') - stbt.wait_for_match(template) + import cv2, stbt + template = cv2.imread("${testdir}/videotestsrc-ball.png") + assert template is not None + for frame in stbt.frames(timeout_secs=20): + stbt.match(template, frame) EOF mkfifo fifo || fail "Initial test setup failed" stbt run -v \ - --source-pipeline 'videotestsrc is-live=true' \ + --source-pipeline "videotestsrc is-live=true pattern=ball ! \ + video/x-raw,width=640" \ --sink-pipeline 'gdppay ! filesink location=fifo' \ + --save-screenshot never \ test.py & test_script=$! trap "kill $test_script; rm fifo" EXIT cat > verify.py <<-EOF && import stbt - stbt.save_frame(stbt.get_frame(), "test.png") stbt.wait_for_match("${testdir}/custom-image-label.png") EOF stbt run -v --control none \ --source-pipeline 'filesrc location=fifo ! gdpdepay' \ + --save-video video.webm \ verify.py } @@ -529,22 +517,6 @@ stbt run -v --control none test.py } -test_that_transformation_pipeline_transforms_video() { - set_config global.transformation_pipeline \ - "videoflip method=horizontal-flip" - cat > test.py <<-EOF - wait_for_match( - "$testdir/videotestsrc-redblue-flipped.png", timeout_secs=0) - EOF - stbt run -v test.py || fail "Video was not flipped" - - cat > test.py <<-EOF - wait_for_match( - "$testdir/videotestsrc-redblue.png", timeout_secs=0) - EOF - ! stbt run -v test.py || fail "Test invalid, shouldn't have matched" -} - test_multithreaded() { cat > test.py <<-EOF && import sys, time @@ -557,11 +529,15 @@ # Kick off the threads pool = ThreadPool(processes=2) - result_iter = pool.imap_unordered(apply, [ - lambda: wait_for_motion(timeout_secs=2), - lambda: wait_for_match( - "$testdir/videotestsrc-checkers-8.png", timeout_secs=2) - ]) + result_iter = pool.imap_unordered( + lambda f: f(), + [ + lambda: stbt.wait_for_motion(timeout_secs=2), + lambda: stbt.wait_for_match( + "$testdir/videotestsrc-checkers-8.png", + timeout_secs=2), + ], + chunksize=1) # Change the pattern stbt.press(sys.argv[1]) @@ -569,9 +545,9 @@ # See which matched result = result_iter.next() if isinstance(result, MotionResult): - print "Motion" + print("Motion") elif isinstance(result, MatchResult): - print "Checkers" + print("Checkers") EOF stbt run -v test.py checkers-8 >out.log @@ -589,7 +565,7 @@ ts = set() for _ in range(10): ts.add(stbt.get_frame().time) - print "Saw %i unique frames" % len(ts) + print("Saw %i unique frames" % len(ts)) assert len(ts) < 5 EOF stbt run test.py || fail "Incorrect get_frame() behaviour" @@ -597,22 +573,26 @@ test_that_two_frames_iterators_can_return_the_same_frames_as_each_other() { cat > test.py <<-EOF && - import itertools + try: + from itertools import izip as zip + except ImportError: + pass sa = set() sb = set() - for a, b in itertools.izip(stbt.frames(), stbt.frames()): - if len(sa) >= 10: + for a, b in zip(stbt.frames(), stbt.frames()): + if len(sa) >= 2: break sa.add(a.time) sb.add(b.time) - print sorted(sa) - print sorted(sb) - assert len(sa) == 10 - assert len(sb) == 10 + print(sorted(sa)) + print(sorted(sb)) # sa and sb contain the same frames: assert sa == sb EOF - stbt run -vv test.py || fail "Incorrect frames() behaviour" + stbt run -vv \ + --source-pipeline="videotestsrc is-live=true ! \ + video/x-raw,format=BGR,width=320,height=240,framerate=2/1" \ + test.py || fail "Incorrect frames() behaviour" } test_that_press_returns_a_pressresult() { diff -Nru stb-tester-30-5-gbefe47c/tests/test-stbt-run.sh stb-tester-31/tests/test-stbt-run.sh --- stb-tester-30-5-gbefe47c/tests/test-stbt-run.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test-stbt-run.sh 2019-09-18 14:04:32.000000000 +0000 @@ -21,7 +21,7 @@ touch module.py cat > test.py <<-EOF import module - print '__file__: ' + __file__ + print('__file__: ' + __file__) assert __file__ == "test.py" EOF stbt run -v test.py @@ -112,7 +112,7 @@ ! stbt run -v test.py && [ -f screenshot.png ] && ! [ -f thumbnail.jpg ] && - python <<-EOF + $python <<-EOF import cv2, numpy ss = cv2.imread('screenshot.png') gf = cv2.imread('grabbed-frame.png') @@ -131,9 +131,9 @@ from gi.repository import GLib for c in range(5, 0, -1): - print "%i bottles of beer on the wall" % c + print("%i bottles of beer on the wall" % c) time.sleep(1) - print "No beer left" + print("No beer left") EOF stbt run test.py & STBT_PID=$! @@ -168,7 +168,7 @@ mkdir tests cat >tests/helpers.py <<-EOF def my_helper(): - print "my_helper() called" + print("my_helper() called") open("touched", "w").close() EOF cat >tests/test.py <<-EOF @@ -218,54 +218,107 @@ || fail "Relative imports in package should work" } -check_unicode_error() { - cat >expected.log <<-EOF - Saved screenshot to 'screenshot.png'. - FAIL: test.py: AssertionError: ü - Traceback (most recent call last): - yield - test_function.call() - execfile(filename, test_globals) - assert False, $u"ü" - AssertionError: ü - EOF - cat expected.log | while read line; do - grep -q -F -e "$line" mylog || fail "Didn't find line: $line" - done -} - test_that_stbt_run_can_print_exceptions_with_unicode_characters() { which unbuffer &>/dev/null || skip "unbuffer is not installed" cat > test.py <<-EOF - # coding: utf-8 - assert False, u"ü" - EOF - - stbt run test.py &> mylog - u="u" assert check_unicode_error - - # We use unbuffer here to provide a tty to `stbt run` to simulate - # interactive use. - LANG=C.UTF-8 unbuffer bash -c 'stbt run test.py' &> mylog - u="u" assert check_unicode_error + # coding:utf-8 + import sys + print(sys.argv[1]) + if sys.argv[1] == "raise-unicode": + raise RuntimeError(u"Röthlisberger") + elif sys.argv[1] == "raise-bytes": + raise RuntimeError(u"Röthlisberger".encode("utf-8")) + elif sys.argv[1] == "raise-non-utf8-bytes": + # https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + raise RuntimeError(b"\xfe") + elif sys.argv[1] == "assert-unicode": + assert False, u"Röthlisberger" + elif sys.argv[1] == "assert-bytes": + assert False, u"Röthlisberger".encode("utf-8") + elif sys.argv[1] == "assert-without-message": + assert u"Röthlisberger" is None + EOF + + unicode_test raise-unicode + unicode_test raise-bytes + unicode_test raise-non-utf8-bytes + unicode_test assert-unicode + unicode_test assert-bytes + unicode_test assert-without-message +} + +unicode_test() { + echo "unicode_test $1: writing to file" + stbt run test.py $1 &> $1.log + assert check_unicode_error $1 $1.log + + echo "unicode_test $1: using unbuffer to simulate interactive use" + unbuffer bash -c "stbt run test.py $1" &> $1.unbuffer.log + assert check_unicode_error $1 $1.unbuffer.log } -test_that_stbt_run_can_print_exceptions_with_encoded_utf8_string() { - which unbuffer &>/dev/null || skip "unbuffer is not installed" - - cat > test.py <<-EOF - # coding: utf-8 - assert False, "ü" - EOF - - stbt run test.py &> mylog - assert check_unicode_error +check_unicode_error() { + local scenario=$1 output=$2 - # We use unbuffer here to provide a tty to `stbt run` to simulate - # interactive use. - LANG=C.UTF-8 unbuffer bash -c 'stbt run test.py' &> mylog - assert check_unicode_error + case "$python_version,$scenario" in + *,raise-unicode|2.7,raise-bytes) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: RuntimeError: Röthlisberger + Traceback (most recent call last): + File ".*/test.py", line .*, in + raise RuntimeError(u"Röthlisberger".*) + RuntimeError: Röthlisberger + EOF + 3,raise-bytes) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: RuntimeError: b'R.xc3.xb6thlisberger' + Traceback (most recent call last): + File ".*/test.py", line .*, in + raise RuntimeError(u"Röthlisberger".encode("utf-8")) + RuntimeError: b'R.xc3.xb6thlisberger' + EOF + # It looks like "RuntimeError: �" but I can't get grep to match it: + *,raise-non-utf8-bytes) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: RuntimeError: + Traceback (most recent call last): + File ".*/test.py", line .*, in + raise RuntimeError(b".xfe") + RuntimeError: + EOF + *,assert-unicode|2.7,assert-bytes) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: AssertionError: Röthlisberger + Traceback (most recent call last): + File ".*/test.py", line .*, in + assert False, u"Röthlisberger".* + AssertionError: Röthlisberger + EOF + 3,assert-bytes) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: AssertionError: b'R.xc3.xb6thlisberger' + Traceback (most recent call last): + File ".*/test.py", line .*, in + assert False, u"Röthlisberger".encode("utf-8") + AssertionError: b'R.xc3.xb6thlisberger' + EOF + *,assert-without-message) cat <<-EOF ;; + Saved screenshot to 'screenshot.png'. + FAIL: test.py: AssertionError: assert u"Röthlisberger" is None + Traceback (most recent call last): + File ".*test.py", line .*, in + assert u"Röthlisberger" is None + AssertionError + EOF + *) fail "check_unicode_error isn't implemented for $scenario" ;; + esac | + while IFS='' read line; do + grep -q -e "^$line" $output || { + cat $output; + fail "Didn't find line: «$line»"; + } + done } test_that_error_control_raises_exception() { @@ -289,3 +342,21 @@ ! stbt run -v test.py && grep -q 'FAIL: test.py: RuntimeError: No remote control configured' log } + +test_that_stbt_run_doesnt_import_unrelated_files() { + # This is a regression test for a bug where if you import from + # 'future.moves' (a Python 2/3 compatibility library) it automatically + # tries to import every python module it finds at the current working + # directory. + cat > test.py <<-EOF + print("This is coming from test.py") + EOF + cat > test2.py <<-EOF + import stbt + print("OK") + EOF + stbt run -v test2.py || fail + if grep -q "This is coming from test.py" log; then + fail "test.py was run but I asked for test2.py" + fi +} diff -Nru stb-tester-30-5-gbefe47c/tests/test_success.py stb-tester-31/tests/test_success.py --- stb-tester-30-5-gbefe47c/tests/test_success.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_success.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -if __name__ == '__main__': - pass diff -Nru stb-tester-30-5-gbefe47c/tests/test_transition.py stb-tester-31/tests/test_transition.py --- stb-tester-30-5-gbefe47c/tests/test_transition.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/test_transition.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,3 +1,8 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import time import cv2 @@ -16,7 +21,7 @@ def press(self, key): from _stbt.core import _Keypress - frame_before = self.frames().next() + frame_before = next(self.frames()) self.state = key return _Keypress(key, time.time(), time.time(), frame_before) @@ -59,7 +64,7 @@ _stbt = FakeDeviceUnderTest() transition = stbt.press_and_wait("white", stable_secs=0.1, _dut=_stbt) - print transition + print(transition) assert transition assert transition.status == stbt.TransitionStatus.COMPLETE assert transition.press_time < transition.animation_start_time @@ -69,7 +74,7 @@ transition = stbt.press_and_wait("fade-to-black", stable_secs=0.1, _dut=_stbt) - print transition + print(transition) assert transition assert transition.status == stbt.TransitionStatus.COMPLETE assert transition.animation_start_time < transition.end_time @@ -79,7 +84,7 @@ def test_press_and_wait_start_timeout(): transition = stbt.press_and_wait("black", timeout_secs=0.2, stable_secs=0.1, _dut=FakeDeviceUnderTest()) - print transition + print(transition) assert not transition assert transition.status == stbt.TransitionStatus.START_TIMEOUT @@ -87,13 +92,13 @@ def test_press_and_wait_stable_timeout(): transition = stbt.press_and_wait("ball", timeout_secs=0.2, stable_secs=0.1, _dut=FakeDeviceUnderTest()) - print transition + print(transition) assert not transition assert transition.status == stbt.TransitionStatus.STABLE_TIMEOUT transition = stbt.press_and_wait("ball", stable_secs=0, _dut=FakeDeviceUnderTest()) - print transition + print(transition) assert transition assert transition.status == stbt.TransitionStatus.COMPLETE @@ -111,7 +116,7 @@ transition = stbt.press_and_wait( "ball", mask=mask, region=region, timeout_secs=0.2, stable_secs=0.1, _dut=FakeDeviceUnderTest()) - print transition + print(transition) assert transition.status == expected @@ -124,7 +129,7 @@ _stbt.press("ball") transition = stbt.wait_for_transition_to_end( timeout_secs=0.2, stable_secs=0.1, _dut=_stbt) - print transition + print(transition) assert not transition assert transition.status == stbt.TransitionStatus.STABLE_TIMEOUT @@ -134,7 +139,7 @@ ["black"] * 10 + ["fade-to-white"] * 2 + ["white"] * 100) transition = stbt.press_and_wait("fade-to-white", _dut=_stbt) - print transition + print(transition) assert transition assert isclose(transition.animation_start_time, transition.press_time + 0.40, diff -Nru stb-tester-30-5-gbefe47c/tests/timeout.pl stb-tester-31/tests/timeout.pl --- stb-tester-30-5-gbefe47c/tests/timeout.pl 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/timeout.pl 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -#!/usr/bin/env perl - -# Portable timeout command. Usage: timeout [...] - -alarm shift @ARGV; -exec @ARGV; -print "timeout: command not found: @ARGV\n"; -exit 1; diff -Nru stb-tester-30-5-gbefe47c/tests/utils.sh stb-tester-31/tests/utils.sh --- stb-tester-30-5-gbefe47c/tests/utils.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/utils.sh 2019-09-18 14:04:32.000000000 +0000 @@ -1,6 +1,4 @@ -# Portable timeout command. Usage: timeout [...] -timeout() { "$testdir"/timeout.pl "$@"; } -timedout=142 +timedout=124 fail() { echo "error: $*"; exit 1; } skip() { echo "skipping: $*"; exit 77; } @@ -27,7 +25,7 @@ } set_config() { - PYTHONPATH=$srcdir python - "$@" <<-EOF + PYTHONPATH=$srcdir $python - "$@" <<-EOF import sys, _stbt.config section, name = sys.argv[1].split('.') _stbt.config.set_config(section, name, sys.argv[2]) diff -Nru stb-tester-30-5-gbefe47c/tests/validate-ocr.py stb-tester-31/tests/validate-ocr.py --- stb-tester-30-5-gbefe47c/tests/validate-ocr.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/validate-ocr.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python u""" validate-ocr.py can be run on a corpus of test images reporting how good a job @@ -29,6 +29,11 @@ stb-tester git tree to allow corpuses containing screen captures from many set-top boxes without bloating the main stb-tester repo or risking upsetting the owners of the various set-top box UIs. """ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import * # pylint:disable=redefined-builtin,unused-wildcard-import,wildcard-import,wrong-import-order import argparse import os diff -Nru stb-tester-30-5-gbefe47c/tests/webminspector/webminspector.py stb-tester-31/tests/webminspector/webminspector.py --- stb-tester-30-5-gbefe47c/tests/webminspector/webminspector.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/tests/webminspector/webminspector.py 2019-09-18 14:04:32.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python ## ## Copyright (c) 2010 The WebM project authors. All Rights Reserved. ## diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/ChangeLog stb-tester-31/vendor/py-lmdb/ChangeLog --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/ChangeLog 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/ChangeLog 1970-01-01 00:00:00.000000000 +0000 @@ -1,209 +0,0 @@ - -2015-06-07 v0.86 - -* LMDB_FORCE_SYSTEM builds were broken by the GIL/page fault change. This - release fixes the problem. - -* Various cosmetic fixes to documentation. - - -2015-06-06 v0.85 - -* New exception class: lmdb.BadDbiError. - -* Environment.copy() and Environment.copyfd() now support compact=True, to - trigger database compaction while copying. - -* Various small documentation updates. - -* CPython set_range_dup() and set_key_dup() both invoked MDB_GET_BOTH, however - set_range_dup() should have instead invoked MDB_GET_BOTH_RANGE. Fix by - Matthew Battifarano. - -* lmdb.tool module was broken on Win32, since Win32 lacks signal.SIGWINCH. Fix - suggested by David Khess. - -* LMDB 0.9.14 is bundled along with extra fixes from mdb.RE/0.9 (release - engineering) branch. - -* CPython previously lacked a Cursor.close() method. Problem was noticed by - Jos Vos. - -* Several memory leaks affecting the CFFI implementation when running on - CPython were fixed, apparent only when repeatedly opening and discarding a - large number of environments. Noticed by Jos Vos. - -* The CPython extension previously did not support weakrefs on Environment - objects, and the implementation for Transaction objects was flawed. The - extension now correctly invalidates weakrefs during deallocation. - -* Both variants now try to avoid taking page faults with the GIL held, - accomplished by touching one byte of every page in a value during reads. - This does not guarantee faults will never occur with the GIL held, but it - drastically reduces the possibility. The binding should now be suitable for - use in multi-threaded applications with databases containing >2KB values - where the entire database does not fit in RAM. - - -2014-09-22 v0.84 - -* LMDB 0.9.14 is bundled. - -* CFFI Cursor.putmulti() could crash when append=False and a key already - existed. - - -2014-06-24 v0.83 - -* LMDB 0.9.13 is bundled along with extra fixes from upstream Git. - -* Environment.__enter__() and __exit__() are implemented, allowing - Environments to behave like context managers. - -* Cursor.close(), __enter__() and __exit__() are implemented, allowing Cursors - to be explicitly closed. In CFFI this mechanism *must* be used when many - cursors are used within a single transaction, otherwise a resource leak will - occur. - -* Dependency tracking in CFFI is now much faster, especially on PyPy, however - at a cost: Cursor use must always be wrapped in a context manager, or - .close() must be manually invoked for discarded Cursors when the parent - transaction is long lived. - -* Fixed crash in CFFI Cursor.putmulti(). - - -2014-05-26 v0.82 - -* Both variants now implement max_spare_txns, reducing the cost of creating a - read-only transaction 4x for an uncontended database and by up to 20x for - very read-busy environments. By default only 1 read-only transaction is - cached, adjust max_spare_txns= parameter if your script operates multiple - simultaneous read transactions. - -* Patch from Vladimir Vladimirov implementing MDB_NOLOCK. - -* The max_spare_iters and max_spare_cursors parameters were removed, neither - ever had any effect. - -* Cursor.putmulti() implemented based on a patch from Luke Kenneth Casson - Leighton. This function moves the loop required to batch populate a - database out of Python and into C. - -* The bundled LMDB 0.9.11 has been updated with several fixes from upstream - Git. - -* The cost of using keyword arguments in the CPython extension was - significantly reduced. - - -2014-04-26 v0.81 - -* On Python 2.x the extension module would silently interpret Unicode - instances as buffer objects, causing UCS-2/UCS-4 string data to end up in - the database. This was never intentional and now raises TypeError. Any - Unicode data passed to py-lmdb must explicitly be encoded with .encode() - first. - -* open_db()'s name argument was renamed to key, and its semantics now match - get() and put(): in other words the key must be a bytestring, and passing - Unicode will raise TypeError. - -* The extension module now builds under Python 3.4 on Windows. - - -2014-04-21 v0.80 - -* Both variants now build successfully as 32 bit / 64bit binaries on - Windows under Visual Studio 9.0, the compiler for Python 2.7. This enables - py-lmdb to be installed via pip on Windows without requiring a compiler to - be available. In future, .egg/.whl releases will be pre-built for all recent - Python versions on Windows. - - Known bugs: Environment.copy() and Environment.copyfd() currently produce a - database that cannot be reopened. - -* The lmdb.enable_drop_gil() function was removed. Its purpose was - experimental at best, confusing at worst. - - -2014-03-17 v0.79 - -* CPython Cursor.delete() lacked dupdata argument, fixed. - -* Fixed minor bug where CFFI _get_cursor() did not note its idea of - the current key and value were up to date. - -* Cursor.replace() and Cursor.pop() updated for MDB_DUPSORT databases. For - pop(), the first data item is popped and returned. For replace(), the first - data item is returned, and all duplicates for the key are replaced. - -* Implement remaining Cursor methods necessary for working with MDB_DUPSORT - databases: next_dup(), next_nodup(), prev_dup(), prev_nodup(), first_dup(), - last_dup(), set_key_dup(), set_range_dup(), iternext_dup(), - iternext_nodup(), iterprev_dup(), iterprev_nodup(). - -* The default for Transaction.put(dupdata=...) and Cursor.put(dupdata=...) has - changed from False to True. The previous default did not reflect LMDB's - normal mode of operation. - -* LMDB 0.9.11 is bundled along with extra fixes from upstream Git. - - -2014-01-18 v0.78 - -* Patch from bra-fsn to fix LMDB_LIBDIR. - -* Various inaccurate documentation improvements. - -* Initial work towards Windows/Microsoft Visual C++ 9.0 build. - -* LMDB 0.9.11 is now bundled. - -* To work around install failures minimum CFFI version is now >=0.8.0. - -* ticket #38: remove all buffer object hacks. This results in ~50% slowdown - for cursor enumeration, but results in far simpler object lifetimes. A - future version may introduce a better mechanism for achieving the same - performance without loss of sanity. - - -2013-11-30 v0.77 - -* Added Environment.max_key_size(), Environment.max_readers(). - -* CFFI now raises the correct Error subclass associated with an MDB_* return - code. - -* Numerous CFFI vs. CPython behavioural inconsistencies have been fixed. - -* An endless variety of Unicode related 2.x/3.x/CPython/CFFI fixes were made. - -* LMDB 0.9.10 is now bundled, along with some extra fixes from Git. - -* Added Environment(meminit=...) option. - - -2013-10-28 v0.76 - -* Added support for Environment(..., readahead=False). - -* LMDB 0.9.9 is now bundled. - -* Many Python 2.5 and 3.x fixes were made. Future changes are automatically - tested via Travis CI . - -* When multiple cursors exist, and one cursor performs a mutation, - remaining cursors may have returned corrupt results via key(), value(), - or item(). Mutations are now explicitly tracked and cause the cursor's - data to be refreshed in this case. - -* setup.py was adjusted to ensure the distutils default of '-DNDEBUG' is never - defined while building LMDB. This caused many important checks in the engine - to be disabled. - -* The old 'transactionless' API was removed. A future version may support the - same API, but the implementation will be different. - -* Transaction.pop() and Cursor.pop() helpers added, to complement - Transaction.replace() and Cursor.replace(). diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/conf.py stb-tester-31/vendor/py-lmdb/docs/conf.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/conf.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/conf.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,280 +0,0 @@ -# -*- coding: utf-8 -*- -# -# lmdb documentation build configuration file, created by -# sphinx-quickstart on Tue Feb 5 00:39:26 2013. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] - -intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'lmdb' -copyright = u'2013, David Wilson' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# - -def grep_version(): - path = os.path.join(os.path.dirname(__file__), '../lmdb/__init__.py') - with open(path) as fp: - for line in fp: - if line.startswith('__version__'): - return eval(line.split()[-1]) - -# The short X.Y version. -version = grep_version() -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'acid' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - 'github_repo': 'https://github.com/dw/py-lmdb/' -} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['themes'] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -html_use_index = False - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -html_show_sourcelink = False - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -html_show_sphinx = False - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'lmdbdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'lmdb.tex', u'lmdb Documentation', - u'David Wilson', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'lmdb', u'lmdb Documentation', - [u'David Wilson'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'lmdb', u'lmdb Documentation', - u'David Wilson', 'lmdb', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - - - -import sys - -class Mock(object): - def __init__(self, *args, **kwargs): - pass - - def __call__(self, *args, **kwargs): - return Mock() - - @classmethod - def __getattr__(cls, name): - if name in ('__file__', '__path__'): - return '/dev/null' - elif 0 and name[0] == name[0].upper(): - mockType = type(name, (), {}) - mockType.__module__ = __name__ - return mockType - else: - return Mock() - -MOCK_MODULES = ['cffi'] -for mod_name in MOCK_MODULES: - sys.modules[mod_name] = Mock() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/index.rst stb-tester-31/vendor/py-lmdb/docs/index.rst --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/index.rst 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/index.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,669 +0,0 @@ - -lmdb -==== - -.. currentmodule:: lmdb - -.. toctree:: - :hidden: - :maxdepth: 2 - -This is a universal Python binding for the `LMDB 'Lightning' Database -`_. Two variants are provided and automatically selected -during install: a `CFFI `_ variant -that supports `PyPy `_ and all versions of CPython >=2.6, -and a C extension that supports CPython 2.5-2.7 and >=3.3. Both variants -provide the same interface. - -LMDB is a tiny database with some excellent properties: - -* Ordered map interface (keys are always lexicographically sorted). -* Reader/writer transactions: readers don't block writers, writers don't block - readers. Each environment supports one concurrent write transaction. -* Read transactions are extremely cheap. -* Environments may be opened by multiple processes on the same host, making it - ideal for working around Python's `GIL - `_. -* Multiple named databases may be created with transactions covering all - named databases. -* Memory mapped, allowing for zero copy lookup and iteration. This is - optionally exposed to Python using the :py:func:`buffer` interface. -* Maintenance requires no external process or background threads. -* No application-level caching is required: LMDB fully exploits the operating - system's buffer cache. - - -Installation: Windows -+++++++++++++++++++++ - -Binary eggs and wheels are published via PyPI for Windows, allowing the binding -to be installed via pip and easy_install without the need for a compiler to be -present. The binary releases statically link against the bundled version of -LMDB. - -Initially 32-bit and 64-bit binaries are provided for Python 2.7; in future -binaries will be published for all supported versions of Python. - -To install, use a command like: - - :: - - C:\Python27\python -mpip install lmdb - -Or: - - :: - - C:\Python27\python -measy_install lmdb - - -Installation: UNIX -++++++++++++++++++ - -For convenience, a supported version of LMDB is bundled with the binding and -built statically by default. If your system distribution includes LMDB, set the -``LMDB_FORCE_SYSTEM`` environment variable, and optionally ``LMDB_INCLUDEDIR`` -and ``LMDB_LIBDIR`` prior to invoking ``setup.py``. - -The CFFI variant depends on CFFI, which in turn depends on ``libffi``, which -may need to be installed from a package. On CPython, both variants additionally -depend on the CPython development headers. On Debian/Ubuntu: - - :: - - apt-get install libffi-dev python-dev build-essential - -To install the C extension, ensure a C compiler and `pip` or `easy_install` are -available and type: - - :: - - pip install lmdb - # or - easy_install lmdb - -The CFFI variant may be used on CPython by setting the ``LMDB_FORCE_CFFI`` -environment variable before installation, or before module import with an -existing installation: - - :: - - >>> import os - >>> os.environ['LMDB_FORCE_CFFI'] = '1' - - >>> # CFFI variant is loaded. - >>> import lmdb - - -Getting Help -++++++++++++ - -Before getting in contact, please ensure you have thoroughly reviewed this -documentation, and if applicable, the associated -`official Doxygen documentation `_. - -If you have found a bug, please report it on the `GitHub issue tracker -`_, or mail it to the list below if -you're allergic to GitHub. - -For all other problems and related discussion, please direct it to -`the py-lmdb@freelists.org mailing list `_. -You must be subscribed to post. The `list archives -`_ are also available. - - -Named Databases -+++++++++++++++ - -Named databases require the `max_dbs=` parameter to be provided when calling -:py:func:`lmdb.open` or :py:class:`lmdb.Environment`. This must be done by the -first process or thread opening the environment. - -Once a correctly configured :py:class:`Environment` is created, new named -databases may be created via :py:meth:`Environment.open_db`. - - -Storage efficiency & limits -+++++++++++++++++++++++++++ - -Records are grouped into pages matching the operating system's VM page size, -which is usually 4096 bytes. Each page must contain at least 2 records, in -addition to 8 bytes per record and a 16 byte header. Due to this the engine is -most space-efficient when the combined size of any (8+key+value) combination -does not exceed 2040 bytes. - -When an attempt to store a record would exceed the maximum size, its value part -is written separately to one or more dedicated pages. Since the trailer of the -last page containing the record value cannot be shared with other records, it -is more efficient when large values are an approximate multiple of 4096 bytes, -minus 16 bytes for an initial header. - -Space usage can be monitored using :py:meth:`Environment.stat`: - - :: - - >>> pprint(env.stat()) - {'branch_pages': 1040L, - 'depth': 4L, - 'entries': 3761848L, - 'leaf_pages': 73658L, - 'overflow_pages': 0L, - 'psize': 4096L} - -This database contains 3,761,848 records and no values were spilled -(``overflow_pages``). - -By default record keys are limited to 511 bytes in length, however this can be -adjusted by rebuilding the library. The compile-time key length can be queried -via :py:meth:`Environment.max_key_size()`. - - -Memory usage -++++++++++++ - -Diagnostic tools often overreport the memory usage of LMDB databases, since the -tools poorly classify that memory. The Linux ``ps`` command ``RSS`` measurement -may report a process as having an entire database resident, causing user alarm. -While the entire database may really be resident, it is half the story. - -Unlike heap memory, pages in file-backed memory maps, such as those used by -LMDB, may be efficiently reclaimed by the OS at any moment so long as the pages -in the map are `clean`. `Clean` simply means that the resident pages' contents -match the associated pages that live in the disk file that backs the mapping. A -clean mapping works exactly like a cache, and in fact it is a cache: the `OS -page cache `_. - -On Linux, the ``/proc//smaps`` file contains one section for each memory -mapping in a process. To inspect the actual memory usage of an LMDB database, -look for a ``data.mdb`` entry, then observe its `Dirty` and `Clean` values. - -When no write transaction is active, all pages in an LMDB database should be -marked `clean`, unless the Environment was opened with `sync=False`, and no -explicit :py:meth:`Environment.sync` has been called since the last write -transaction, and the OS writeback mechanism has not yet opportunistically -written the dirty pages to disk. - - -Bytestrings -+++++++++++ - -This documentation uses `bytestring` to mean either the Python<=2.7 -:py:func:`str` type, or the Python>=3.0 :py:func:`bytes` type, depending on the -Python version in use. - -Due to the design of Python 2.x, LMDB will happily accept Unicode instances -where :py:func:`str` instances are expected, so long as they contain only ASCII -characters, in which case they are implicitly encoded to ASCII. You should not -rely on this behaviour! It results in brittle programs that often break the -moment they are deployed in production. Always explicitly encode and decode any -Unicode values before passing them to LMDB. - -This documentation uses :py:func:`bytes` in examples. In Python 3.x this is a -distinct type, whereas in Python 2.6 and 2.7 it is simply an alias for -:py:func:`str`. Since Python 2.5 does not have this alias, you should -substitute :py:func:`str` for :py:func:`bytes` in any code examples below when -running on Python 2.5. - - -Buffers -+++++++ - -Since LMDB is memory mapped it is possible to access record data without keys -or values ever being copied by the kernel, database library, or application. To -exploit this the library can be instructed to return :py:func:`buffer` objects -instead of bytestrings by passing `buffers=True` to -:py:meth:`Environment.begin` or :py:class:`Transaction`. - -In Python :py:func:`buffer` objects can be used in many places where -bytestrings are expected. In every way they act like a regular sequence: they -support slicing, indexing, iteration, and taking their length. Many Python APIs -will automatically convert them to bytestrings as necessary: - - :: - - >>> txn = env.begin(buffers=True) - >>> buf = txn.get('somekey') - >>> buf - - - >>> len(buf) - 4096 - >>> buf[0] - 'a' - >>> buf[:2] - 'ab' - >>> value = bytes(buf) - >>> len(value) - 4096 - >>> type(value) - - -It is also possible to pass buffers directly to many native APIs, for example -:py:meth:`file.write`, :py:meth:`socket.send`, :py:meth:`zlib.decompress` and -so on. A buffer may be sliced without copying by passing it to -:py:func:`buffer`: - - :: - - >>> # Extract bytes 10 through 210: - >>> sub_buf = buffer(buf, 10, 200) - >>> len(sub_buf) - 200 - -In both PyPy and CPython, returned buffers *must be discarded* after their -producing transaction has completed or been modified in any way. To preserve -buffer's contents, copy it using :py:func:`bytes`: - - .. code-block:: python - - with env.begin(write=True, buffers=True) as txn: - buf = env.get('foo') # only valid until the next write. - buf_copy = bytes(buf) # valid forever - env.delete('foo') # this is a write! - env.put('foo2', 'bar2') # this is also a write! - - print('foo: %r' % (buf,)) # ERROR! invalidated by write - print('foo: %r' % (buf_copy,) # OK - - print('foo: %r' % (buf,)) # ERROR! also invalidated by txn end - print('foo: %r' % (buf_copy,) # still OK - - -``writemap`` mode -+++++++++++++++++ - -When :py:class:`Environment` or :py:func:`open` is invoked with -``writemap=True``, the library will use a writeable memory mapping to directly -update storage. This improves performance at a cost to safety: it is possible -(though fairly unlikely) for buggy C code in the Python process to accidentally -overwrite the map, resulting in database corruption. - -.. caution:: - - This option may cause filesystems that don't support sparse files, such as - OSX, to immediately preallocate `map_size=` bytes of underlying storage. - - -Resource Management -+++++++++++++++++++ - -:py:class:`Environment`, :py:class:`Transaction`, and :py:class:`Cursor` -support the context manager protocol, allowing for robust resource cleanup in -the case of exceptions. - -.. code-block:: python - - with env.begin() as txn: - with txn.cursor() as curs: - # do stuff - print 'key is:', curs.get('key') - -On CFFI it is important to use the :py:class:`Cursor` context manager, or -explicitly call :py:meth:`Cursor.close` if many cursors are created within a -single transaction. Failure to close a cursor on CFFI may cause many dead -objects to accumulate until the parent transaction is aborted or committed. - - -Transaction management -++++++++++++++++++++++ - -While any reader exists, writers cannot reuse space in the database file that -has become unused in later versions. Due to this, continual use of long-lived -read transactions may cause the database to grow without bound. A lost -reference to a read transaction will simply be aborted (and its reader slot -freed) when the :py:class:`Transaction` is eventually garbage collected. This -should occur immediately on CPython, but may be deferred indefinitely on PyPy. - -However the same is *not* true for write transactions: losing a reference to a -write transaction can lead to deadlock, particularly on PyPy, since if the same -process that lost the :py:class:`Transaction` reference immediately starts -another write transaction, it will deadlock on its own lock. Subsequently the -lost transaction may never be garbage collected (since the process is now -blocked on itself) and the database will become unusable. - -These problems are easily avoided by always wrapping :py:class:`Transaction` in -a ``with`` statement somewhere on the stack: - -.. code-block:: python - - # Even if this crashes, txn will be correctly finalized. - with env.begin() as txn: - if txn.get('foo'): - function_that_stashes_away_txn_ref(txn) - function_that_leaks_txn_refs(txn) - crash() - - -Threads -+++++++ - -``MDB_NOTLS`` mode is used exclusively, which allows read transactions to -freely migrate across threads and for a single thread to maintain multiple read -transactions. This enables mostly care-free use of read transactions, for -example when using `gevent `_. - -Most objects can be safely called by a single caller from a single thread, and -usually it only makes sense to to have a single caller, except in the case of -:py:class:`Environment`. - -Most :py:class:`Environment` methods are thread-safe, and may be called -concurrently, except for :py:meth:`Environment.close`. - -A write :py:class:`Transaction` may only be used from the thread it was created -on. - -A read-only :py:class:`Transaction` can move across threads, but it cannot be -used concurrently from multiple threads. - -:py:class:`Cursor` is not thread-safe, but it does not make sense to use it on -any thread except the thread that currently owns its associated -:py:class:`Transaction`. - - -Interface -+++++++++ - -.. py:function:: lmdb.open(path, **kwargs) - - Shortcut for :py:class:`Environment` constructor. - -.. autofunction:: lmdb.version - - -Environment class -################# - -.. autoclass:: lmdb.Environment - :members: - - -Transaction class -################# - -.. autoclass:: lmdb.Transaction - :members: - - -Cursor class -############ - -.. autoclass:: lmdb.Cursor - :members: - - -Exceptions -########## - -.. autoclass:: lmdb.Error () -.. autoclass:: lmdb.KeyExistsError () -.. autoclass:: lmdb.NotFoundError () -.. autoclass:: lmdb.PageNotFoundError () -.. autoclass:: lmdb.CorruptedError () -.. autoclass:: lmdb.PanicError () -.. autoclass:: lmdb.VersionMismatchError () -.. autoclass:: lmdb.InvalidError () -.. autoclass:: lmdb.MapFullError () -.. autoclass:: lmdb.DbsFullError () -.. autoclass:: lmdb.ReadersFullError () -.. autoclass:: lmdb.TlsFullError () -.. autoclass:: lmdb.TxnFullError () -.. autoclass:: lmdb.CursorFullError () -.. autoclass:: lmdb.PageFullError () -.. autoclass:: lmdb.MapResizedError () -.. autoclass:: lmdb.IncompatibleError () -.. autoclass:: lmdb.BadDbiError () -.. autoclass:: lmdb.BadRslotError () -.. autoclass:: lmdb.BadTxnError () -.. autoclass:: lmdb.BadValsizeError () -.. autoclass:: lmdb.ReadonlyError () -.. autoclass:: lmdb.InvalidParameterError () -.. autoclass:: lmdb.LockError () -.. autoclass:: lmdb.MemoryError () -.. autoclass:: lmdb.DiskError () - - -Command line tools -++++++++++++++++++ - -A rudimentary interface to most of the binding's functionality is provided. -These functions are useful for e.g. backup jobs. - -:: - - $ python -mlmdb --help - Usage: python -mlmdb [options] - - Basic tools for working with LMDB. - - copy: Consistent high speed backup an environment. - python -mlmdb copy -e source.lmdb target.lmdb - - copyfd: Consistent high speed backup an environment to stdout. - python -mlmdb copyfd -e source.lmdb > target.lmdb/data.mdb - - drop: Delete one or more named databases. - python -mlmdb drop db1 - - dump: Dump one or more databases to disk in 'cdbmake' format. - Usage: dump [db1=file1.cdbmake db2=file2.cdbmake] - - If no databases are given, dumps the main database to 'main.cdbmake'. - - edit: Add/delete/replace values from a database. - python -mlmdb edit --set key=value --set-file key=/path \ - --add key=value --add-file key=/path/to/file \ - --delete key - - get: Read one or more values from a database. - python -mlmdb get [ [ [..]]] - - readers: Display readers in the lock table - python -mlmdb readers -e /path/to/db [-c] - - If -c is specified, clear stale readers. - - restore: Read one or more database from disk in 'cdbmake' format. - python -mlmdb restore db1=file1.cdbmake db2=file2.cdbmake - - The special db name ":main:" may be used to indicate the main DB. - - rewrite: Re-create an environment using MDB_APPEND - python -mlmdb rewrite -e src.lmdb -E dst.lmdb [ [ ..]] - - If no databases are given, rewrites only the main database. - - shell: Open interactive console with ENV set to the open environment. - - stat: Print environment statistics. - - warm: Read environment into page cache sequentially. - - watch: Show live environment statistics - - Options: - -h, --help show this help message and exit - -e ENV, --env=ENV Environment file to open - -d DB, --db=DB Database to open (default: main) - -r READ, --read=READ Open environment read-only - -S MAP_SIZE, --map_size=MAP_SIZE - Map size in megabytes (default: 10) - -a, --all Make "dump" dump all databases - -T TXN_SIZE, --txn_size=TXN_SIZE - Writes per transaction (default: 1000) - -E TARGET_ENV, --target_env=TARGET_ENV - Target environment file for "dumpfd" - -x, --xxd Print values in xxd format - -M MAX_DBS, --max-dbs=MAX_DBS - Maximum open DBs (default: 128) - --out-fd=OUT_FD "copyfd" command target fd - - Options for "edit" command: - --set=SET List of key=value pairs to set. - --set-file=SET_FILE - List of key pairs to read from files. - --add=ADD List of key=value pairs to add. - --add-file=ADD_FILE - List of key pairs to read from files. - --delete=DELETE List of key=value pairs to delete. - - Options for "readers" command: - -c, --clean Clean stale readers? (default: no) - - Options for "watch" command: - --csv Generate CSV instead of terminal output. - --interval=INTERVAL Interval size (default: 1sec) - --window=WINDOW Average window size (default: 10) - - - -Implementation Notes -++++++++++++++++++++ - - -Iterators -######### - -It was tempting to make :py:class:`Cursor` directly act as an iterator, however -that would require overloading its `next()` method to mean something other than -the natural definition of `next()` on an LMDB cursor. It would additionally -introduce unintuitive state tied to the cursor that does not exist in LMDB: -such as iteration direction and the type of value yielded. - -Instead a separate iterator is produced by `__iter__()`, `iternext()`, and -`iterprev()`, with easily described semantics regarding how they interact with -the cursor. - - -Memsink Protocol -################ - -If the ``memsink`` package is available during installation of the CPython -extension, then the resulting module's :py:class:`Transaction` object will act -as a `source` for the `Memsink Protocol -`_. This is an experimental protocol to -allow extension of LMDB's zero-copy design outward to other C types, without -requiring explicit management by the user. - -This design is a work in progress; if you have an application that would -benefit from it, please leave a comment on the ticket above. - - -Deviations from LMDB API -######################## - -`mdb_dbi_close()`: - This is not exposed since its use is perilous at best. Users must ensure - all activity on the DBI has ceased in all threads before closing the - handle. Failure to do this could result in "impossible" errors, or the DBI - slot becoming reused, resulting in operations being serviced by the wrong - named database. Leaving handles open wastes a tiny amount of memory, which - seems a good price to avoid subtle data corruption. - -:py:meth:`Cursor.replace`, :py:meth:`Cursor.pop`: - There are no native equivalents to these calls, they just implement common - operations in C to avoid a chunk of error prone, boilerplate Python from - having to do the same. - -`mdb_set_compare()`, `mdb_set_dupsort()`: - Neither function is exposed for a variety of reasons. In particular, - neither can be supported safely, since exceptions cannot be propagated - through LMDB callbacks, and can lead to database corruption if used - incorrectly. Secondarily, since both functions are repeatedly invoked for - every single lookup in the LMDB read path, most of the performance benefit - of LMDB is lost by introducing Python interpreter callbacks to its hot path. - - There are a variety of workarounds that could make both functions useful, - but not without either punishing binding users who do not require these - features (especially on CFFI), or needlessly complicating the binding for - what is essentially an edge case. - - In all cases where `mdb_set_compare()` might be useful, use of a special - key encoding that encodes your custom order is usually desirable. See - `issue #79 `_ for example - approaches. - - The answer is not so clear for `mdb_set_dupsort()`, since a custom encoding - there may necessitate wasted storage space, or complicating record decoding - in an application's hot path. Please file a ticket if you think you have a - use for `mdb_set_dupsort()`. - - -Technology -########## - -The binding is implemented twice: once using CFFI, and once as native C -extension. This is since a CFFI binding is necessary for PyPy, but its -performance on CPython is very poor. For good performance on CPython, only -Cython and a native extension are viable options. Initially Cython was used, -however this was abandoned due to the effort and relative mismatch involved -compared to writing a native extension. - - -Invalidation lists -################## - -Much effort has gone into avoiding crashes: when some object is invalidated -(e.g. due to :py:meth:`Transaction.abort`), child objects are updated to ensure -they don't access memory of the no-longer-existent resource, and that they -correspondingly free their own resources. On CPython this is accomplished by -weaving a linked list into all ``PyObject`` structures. This avoids the need to -maintain a separate heap-allocated structure, or produce excess ``weakref`` -objects (which internally simply manage their own lists). - -With CFFI this isn't possible. Instead each object has a ``_deps`` dict that -maps dependent object IDs to the corresponding objects. Weakrefs are avoided -since they are very inefficient on PyPy. Prior to invalidation ``_deps`` is -walked to notify each dependent that the resource is about to disappear. - -Finally, each object may either store an explicit ``_invalid`` attribute and -check it prior to every operation, or rely on another mechanism to avoid the -crash resulting from using an invalidated resource. Instead of performing these -explicit tests continuously, on CFFI a magic -``Some_LMDB_Resource_That_Was_Deleted_Or_Closed`` object is used. During -invalidation, all native handles are replaced with an instance of this object. -Since CFFI cannot convert the magical object to a C type, any attempt to make a -native call will raise ``TypeError`` with a nice descriptive type name -indicating the problem. Hacky but efficient, and mission accomplished. - - -Argument parsing -################ - -The CPython module `parse_args()` may look "special", at best. The alternative -`PyArg_ParseTupleAndKeywords` performs continuous heap allocations and string -copies, resulting in a difference of 10,000 lookups/sec slowdown in a -particular microbenchmark. The 10k/sec slowdown could potentially disappear -given a sufficiently large application, so this decision needs revisited at -some stage. - - -ChangeLog -+++++++++ - -.. include:: ../ChangeLog - :literal: - - -License -+++++++ - -.. include:: ../LICENSE - :literal: - - -.. raw:: html - - - diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/Makefile stb-tester-31/vendor/py-lmdb/docs/Makefile --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/Makefile 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/lmdb.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/lmdb.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/lmdb" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/lmdb" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/conf.py.conf stb-tester-31/vendor/py-lmdb/docs/themes/acid/conf.py.conf --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/conf.py.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/themes/acid/conf.py.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys, os -extensions = [] -templates_path = ['{{ template_dir }}', 'templates', '_templates', '.templates'] -source_suffix = '{{ project.suffix }}' -master_doc = 'index' -project = u'{{ project.name }}' -copyright = u'{{ project.copyright }}' -version = '{{ project.version }}' -release = '{{ project.version }}' -exclude_patterns = ['_build'] -pygments_style = 'sphinx' -html_theme = '{{ project.theme }}' -html_theme_path = ['.', '_theme', '.theme'] -htmlhelp_basename = '{{ project.slug }}' -file_insertion_enabled = False -latex_documents = [ - ('index', '{{ project.slug }}.tex', u'{{ project.name }} Documentation', - u'{{ project.copyright }}', 'manual'), -] diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/layout.html stb-tester-31/vendor/py-lmdb/docs/themes/acid/layout.html --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/layout.html 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/themes/acid/layout.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,127 +0,0 @@ - - - -{# Strip out the default script files we provide, but keep some, like mathjax. #} -{% set script_files = [] %} -{%- set url_root = pathto('', 1) %} -{%- if url_root == '#' %}{% set url_root = '' %}{% endif %} -{%- set titlesuffix = " — "|safe + docstitle|e %} - -{%- macro sidebar() %} -

- {%- if logo %} -

- -

- {%- endif %} - - {% if theme_github_repo %} -

- GitHub Repository -

- {% endif %} - -

- Docs for last release -

- -

- Docs for master branch -

- - {%- block sidebartoc %} - {%- include "localtoc.html" %} - {%- endblock %} - {%- block sidebarrel %} - {%- include "relations.html" %} - {%- endblock %} -

-{%- endmacro %} - -{{ title|striptags|e }}{{ titlesuffix }} - - -{%- for cssfile in css_files %} - -{%- endfor %} - - - -{%- for scriptfile in script_files %} - -{%- endfor %} - - -{%- block content %} - {%- block sidebar1 %} {# possible location for sidebar #} {% endblock %} - -

- {%- block document %} -

- {% block body %} {% endblock %} -

- {%- endblock %} - - {%- block sidebar2 %}{{ sidebar() }}{% endblock %} -

-{%- endblock %} - -{% macro relbar() %}{% endmacro %} -{%- block relbar2 %}{{ relbar() }}{% endblock %} - - -

-{%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} - {%- else %} - {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} - {%- endif %} -{%- endif %} -{%- if last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} -{%- endif %} -{%- if show_sphinx %} - {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %} -{%- endif %} -

- -{% if not using_theme %} - {# Keep this here, so that the RTD logo doesn't stomp on the bottom of the theme. #} -
-
-
-{% endif %} - - - - diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/sourcelink.html stb-tester-31/vendor/py-lmdb/docs/themes/acid/sourcelink.html --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/sourcelink.html 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/themes/acid/sourcelink.html 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -{% if 0 %}{% endif %} diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/static/acid.css stb-tester-31/vendor/py-lmdb/docs/themes/acid/static/acid.css --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/static/acid.css 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/themes/acid/static/acid.css 1970-01-01 00:00:00.000000000 +0000 @@ -1,1407 +0,0 @@ -/* Based on rtd.css, copyright 2007-2010 Sphinx team. */ -/* RTD colors - * light blue: #e8ecef - * medium blue: #8ca1af - * dark blue: #465158 - * dark grey: #444444 - * - * white hover: #d1d9df; - * medium blue hover: #697983; - * green highlight: #8ecc4c - * light blue (project bar): #e8ecef - */ - -div.clearer { - clear: both; -} - -div.related { - width: 100%; - font-size: 90%; -} - -div.related h3 { - display: none; -} - -div.related ul { - margin: 0; - padding: 0 0 0 10px; - list-style: none; -} - -div.related li { - display: inline; -} - -div.related li.right { - float: right; - margin-right: 5px; -} - -/* -- sidebar --------------------------------------------------------------- */ - -div.sphinxsidebarwrapper { - padding: 10px 5px 0 10px; -} - -div.sphinxsidebar { - float: left; - width: 250px; - margin-left: -100%; - font-size: 90%; -} - -div.sphinxsidebar ul { - list-style: none; -} - -div.sphinxsidebar ul ul, -div.sphinxsidebar ul.want-points { - margin-left: 20px; - list-style: square; -} - -div.sphinxsidebar ul ul { - margin-top: 0; - margin-bottom: 0; -} - -div.sphinxsidebar form { - margin-top: 10px; -} - -div.sphinxsidebar input { - border: 1px solid #98dbcc; - font-family: sans-serif; - font-size: 1em; -} - -div.sphinxsidebar #searchbox input[type="text"] { - width: 170px; -} - -div.sphinxsidebar #searchbox input[type="submit"] { - width: 30px; -} - -img { - border: 0; -} - -/* -- search page ----------------------------------------------------------- */ - -ul.search { - margin: 10px 0 0 20px; - padding: 0; -} - -ul.search li { - padding: 5px 0 5px 20px; - background-image: url(file.png); - background-repeat: no-repeat; - background-position: 0 7px; -} - -ul.search li a { - font-weight: bold; -} - -ul.search li div.context { - color: #888; - margin: 2px 0 0 30px; - text-align: left; -} - -ul.keywordmatches li.goodmatch a { - font-weight: bold; -} - -/* -- index page ------------------------------------------------------------ */ - -table.contentstable { - width: 90%; -} - -table.contentstable p.biglink { - line-height: 150%; -} - -a.biglink { - font-size: 1.3em; -} - -span.linkdescr { - font-style: italic; - padding-top: 5px; - font-size: 90%; -} - -/* -- general index --------------------------------------------------------- */ - -table.indextable { - width: 100%; -} - -table.indextable td { - text-align: left; - vertical-align: top; -} - -table.indextable dl, table.indextable dd { - margin-top: 0; - margin-bottom: 0; -} - -table.indextable tr.pcap { - height: 10px; -} - -table.indextable tr.cap { - margin-top: 10px; - background-color: #f2f2f2; -} - -img.toggler { - margin-right: 3px; - margin-top: 3px; - cursor: pointer; -} - -div.modindex-jumpbox { - border-top: 1px solid #ddd; - border-bottom: 1px solid #ddd; - margin: 1em 0 1em 0; - padding: 0.4em; -} - -div.genindex-jumpbox { - border-top: 1px solid #ddd; - border-bottom: 1px solid #ddd; - margin: 1em 0 1em 0; - padding: 0.4em; -} - -/* -- general body styles --------------------------------------------------- */ - -a.headerlink { - visibility: hidden; -} - -h1:hover > a.headerlink, -h2:hover > a.headerlink, -h3:hover > a.headerlink, -h4:hover > a.headerlink, -h5:hover > a.headerlink, -h6:hover > a.headerlink, -dt:hover > a.headerlink { - visibility: visible; -} - -div.body p.caption { - text-align: inherit; -} - -div.body td { - text-align: left; -} - -.field-list ul { - padding-left: 1em; -} - -.first { - margin-top: 0 !important; -} - -p.rubric { - margin-top: 30px; - font-weight: bold; -} - -img.align-left, .figure.align-left, object.align-left { - clear: left; - float: left; - margin-right: 1em; -} - -img.align-right, .figure.align-right, object.align-right { - clear: right; - float: right; - margin-left: 1em; -} - -img.align-center, .figure.align-center, object.align-center { - display: block; - margin-left: auto; - margin-right: auto; -} - -.align-left { - text-align: left; -} - -.align-center { - text-align: center; -} - -.align-right { - text-align: right; -} - -/* -- sidebars -------------------------------------------------------------- */ - -div.sidebar { - margin: 0 0 0.5em 1em; - border: 1px solid #ddb; - padding: 7px 7px 0 7px; - background-color: #ffe; - width: 40%; - float: right; -} - -p.sidebar-title { - font-weight: bold; -} - -/* -- topics ---------------------------------------------------------------- */ - -div.topic { - border: 1px solid #ccc; - padding: 7px 7px 0 7px; - margin: 10px 0 10px 0; -} - -p.topic-title { - font-size: 1.1em; - font-weight: bold; - margin-top: 10px; -} - -/* -- admonitions ----------------------------------------------------------- */ - -div.admonition { - margin-top: 10px; - margin-bottom: 10px; - padding: 7px; -} - -div.admonition dt { - font-weight: bold; -} - -div.admonition dl { - margin-bottom: 0; -} - -p.admonition-title { - margin: 0px 10px 5px 0px; - font-weight: bold; -} - -div.body p.centered { - text-align: center; - margin-top: 25px; -} - -/* -- tables ---------------------------------------------------------------- */ - -table.docutils { - border: 0; - border-collapse: collapse; -} - -table.docutils td, table.docutils th { - padding: 1px 8px 1px 5px; - border-top: 0; - border-left: 0; - border-right: 0; - border-bottom: 1px solid #aaa; -} - -table.field-list td, table.field-list th { - border: 0 !important; -} - -table.footnote td, table.footnote th { - border: 0 !important; -} - -th { - text-align: left; - padding-right: 5px; -} - -table.citation { - border-left: solid 1px gray; - margin-left: 1px; -} - -table.citation td { - border-bottom: none; -} - -/* -- other body styles ----------------------------------------------------- */ - -ol.arabic { - list-style: decimal; -} - -ol.loweralpha { - list-style: lower-alpha; -} - -ol.upperalpha { - list-style: upper-alpha; -} - -ol.lowerroman { - list-style: lower-roman; -} - -ol.upperroman { - list-style: upper-roman; -} - -dl { - margin-bottom: 15px; -} - -dd p { - margin-top: 0px; -} - -dd ul, dd table { - margin-bottom: 10px; -} - -dd { - margin-top: 3px; - margin-bottom: 10px; - margin-left: 30px; -} - -dt:target, .highlighted { - background-color: #fbe54e; -} - -dl.glossary dt { - font-weight: bold; - font-size: 1.1em; -} - -.field-list ul { - margin: 0; - padding-left: 1em; -} - -.field-list p { - margin: 0; -} - -.refcount { - color: #060; -} - -.optional { - font-size: 1.3em; -} - -.versionmodified { - font-style: italic; -} - -.system-message { - background-color: #fda; - padding: 5px; - border: 3px solid red; -} - -.footnote:target { - background-color: #ffa; -} - -.line-block { - display: block; - margin-top: 1em; - margin-bottom: 1em; -} - -.line-block .line-block { - margin-top: 0; - margin-bottom: 0; - margin-left: 1.5em; -} - -.guilabel, .menuselection { - font-family: sans-serif; -} - -.accelerator { - text-decoration: underline; -} - -.classifier { - font-style: oblique; -} - -abbr, acronym { - border-bottom: dotted 1px; - cursor: help; -} - -/* -- code displays --------------------------------------------------------- */ - -pre { - overflow: auto; - overflow-y: hidden; /* fixes display issues on Chrome browsers */ -} - -td.linenos pre { - padding: 5px 0px; - border: 0; - background-color: transparent; - color: #aaa; -} - -table.highlighttable { - margin-left: 0.5em; -} - -table.highlighttable td { - padding: 0 0.5em 0 0.5em; -} - -tt.descname { - background-color: transparent; - font-weight: bold; - font-size: 1.2em; -} - -tt.descclassname { - background-color: transparent; -} - -tt.xref, a tt { - background-color: transparent; - font-weight: bold; -} - -h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { - background-color: transparent; -} - -.viewcode-link { - float: right; -} - -.viewcode-back { - float: right; - font-family: sans-serif; -} - -div.viewcode-block:target { - margin: -1px -10px; - padding: 0 10px; -} - -/* -- math display ---------------------------------------------------------- */ - -img.math { - vertical-align: middle; -} - -div.body div.math p { - text-align: center; -} - -span.eqno { - float: right; -} - -/* -- printout stylesheet --------------------------------------------------- */ - -@media print { - div.document, - div.documentwrapper, - div.bodywrapper { - margin: 0 !important; - width: 100%; - } - - div.sphinxsidebar, - div.related, - div.footer, - #top-link { - display: none; - } -} - -/* PAGE LAYOUT -------------------------------------------------------------- */ - -body { - font: 100%/1.5 "ff-meta-web-pro-1","ff-meta-web-pro-2",Arial,"Helvetica Neue",sans-serif; - text-align: center; - color: black; - background-color: #465158; - padding: 0; - margin: 0; -} - -div.document { - text-align: left; - background-color: #e8ecef; -} - -div.bodywrapper { - background-color: #ffffff; - border-left: 1px solid #ccc; - border-bottom: 1px solid #ccc; - margin: 0 0 0 16em; -} - -div.body { - margin: 0; - padding: 0.5em 1.3em; - min-width: 20em; -} - -div.related { - font-size: 1em; - background-color: #465158; -} - -div.documentwrapper { - float: left; - width: 100%; - background-color: #e8ecef; -} - - -/* HEADINGS --------------------------------------------------------------- */ - -h1 { - margin: 0; - padding: 0.7em 0 0.3em 0; - font-size: 1.5em; - line-height: 1.15; - color: #111; - clear: both; -} - -h2 { - margin: 2em 0 0.2em 0; - font-size: 1.35em; - padding: 0; - color: #465158; -} - -h3 { - margin: 1em 0 -0.3em 0; - font-size: 1.2em; - color: #6c818f; -} - -div.body h1 a, div.body h2 a, div.body h3 a, div.body h4 a, div.body h5 a, div.body h6 a { - color: black; -} - -h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor { - display: none; - margin: 0 0 0 0.3em; - padding: 0 0.2em 0 0.2em; - color: #aaa !important; -} - -h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, -h5:hover a.anchor, h6:hover a.anchor { - display: inline; -} - -h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover, -h5 a.anchor:hover, h6 a.anchor:hover { - color: #777; - background-color: #eee; -} - - -/* LINKS ------------------------------------------------------------------ */ - -/* Normal links get a pseudo-underline */ -a { - color: #444; - text-decoration: none; - border-bottom: 1px solid #ccc; -} - -/* Links in sidebar, TOC, index trees and tables have no underline */ -.sphinxsidebar a, -.toctree-wrapper a, -.indextable a, -#indices-and-tables a { - color: #444; - text-decoration: none; - border-bottom: none; -} - -/* Most links get an underline-effect when hovered */ -a:hover, -div.toctree-wrapper a:hover, -.indextable a:hover, -#indices-and-tables a:hover { - color: #111; - text-decoration: none; - border-bottom: 1px solid #111; -} - -/* Footer links */ -div.footer a { - color: #86989B; - text-decoration: none; - border: none; -} -div.footer a:hover { - color: #a6b8bb; - text-decoration: underline; - border: none; -} - -/* Permalink anchor (subtle grey with a red hover) */ -div.body a.headerlink { - color: #ccc; - font-size: 1em; - margin-left: 6px; - padding: 0 4px 0 4px; - text-decoration: none; - border: none; -} -div.body a.headerlink:hover { - color: #c60f0f; - border: none; -} - - -/* NAVIGATION BAR --------------------------------------------------------- */ - -div.related ul { - height: 2.5em; -} - -div.related ul li { - margin: 0; - padding: 0.65em 0; - float: left; - display: block; - color: white; /* For the >> separators */ - font-size: 0.8em; -} - -div.related ul li.right { - float: right; - margin-right: 5px; - color: transparent; /* Hide the | separators */ -} - -/* "Breadcrumb" links in nav bar */ -div.related ul li a { - order: none; - background-color: inherit; - font-weight: bold; - margin: 6px 0 6px 4px; - line-height: 1.75em; - color: #ffffff; - padding: 0.4em 0.8em; - border: none; - border-radius: 3px; -} -/* previous / next / modules / index links look more like buttons */ -div.related ul li.right a { - margin: 0.375em 0; - background-color: #697983; - text-shadow: 0 1px rgba(0, 0, 0, 0.5); - border-radius: 3px; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; -} -/* All navbar links light up as buttons when hovered */ -div.related ul li a:hover { - background-color: #8ca1af; - color: #ffffff; - text-decoration: none; - border-radius: 3px; - -webkit-border-radius: 3px; - -moz-border-radius: 3px; -} -/* Take extra precautions for tt within links */ -a tt, -div.related ul li a tt { - background: inherit !important; - color: inherit !important; -} - - -/* SIDEBAR ---------------------------------------------------------------- */ - -div.sphinxsidebarwrapper { - padding: 0; -} - -div.sphinxsidebar { - margin: 0; - margin-left: -100%; - float: left; - top: 3em; - left: 0; - padding: 0 1em; - width: 14em; - font-size: 1em; - text-align: left; - background-color: #e8ecef; -} - -div.sphinxsidebar img { - max-width: 12em; -} - -div.sphinxsidebar h3, -div.sphinxsidebar h4, -div.sphinxsidebar p.logo { - margin: 1.2em 0 0.3em 0; - font-size: 1em; - padding: 0; - color: #222222; - font-family: "ff-meta-web-pro-1", "ff-meta-web-pro-2", "Arial", "Helvetica Neue", sans-serif; -} - -div.sphinxsidebar h3 a { - color: #444444; -} - -div.sphinxsidebar ul, -div.sphinxsidebar p { - margin-top: 0; - padding-left: 0; - line-height: 130%; - background-color: #e8ecef; -} - -/* No bullets for nested lists, but a little extra indentation */ -div.sphinxsidebar ul ul { - list-style-type: none; - margin-left: 1.5em; - padding: 0; -} - -/* A little top/bottom padding to prevent adjacent links' borders - * from overlapping each other */ -div.sphinxsidebar ul li { - padding: 1px 0; -} - -/* A little left-padding to make these align with the ULs */ -div.sphinxsidebar p.topless { - padding-left: 0 0 0 1em; -} - -/* Make these into hidden one-liners */ -div.sphinxsidebar ul li, -div.sphinxsidebar p.topless { - white-space: nowrap; - overflow: hidden; -} -/* ...which become visible when hovered */ -div.sphinxsidebar ul li:hover, -div.sphinxsidebar p.topless:hover { - overflow: visible; -} - -/* Search text box and "Go" button */ -#searchbox { - margin-top: 2em; - margin-bottom: 1em; - background: #ddd; - padding: 0.5em; - border-radius: 6px; - -moz-border-radius: 6px; - -webkit-border-radius: 6px; -} -#searchbox h3 { - margin-top: 0; -} - -/* Make search box and button abut and have a border */ -input, -div.sphinxsidebar input { - border: 1px solid #999; - float: left; -} - -/* Search textbox */ -input[type="text"] { - margin: 0; - padding: 0 3px; - height: 20px; - width: 144px; - border-top-left-radius: 3px; - border-bottom-left-radius: 3px; - -moz-border-radius-topleft: 3px; - -moz-border-radius-bottomleft: 3px; - -webkit-border-top-left-radius: 3px; - -webkit-border-bottom-left-radius: 3px; -} -/* Search button */ -input[type="submit"] { - margin: 0 0 0 -1px; /* -1px prevents a double-border with textbox */ - height: 22px; - color: #444; - background-color: #e8ecef; - padding: 1px 4px; - font-weight: bold; - border-top-right-radius: 3px; - border-bottom-right-radius: 3px; - -moz-border-radius-topright: 3px; - -moz-border-radius-bottomright: 3px; - -webkit-border-top-right-radius: 3px; - -webkit-border-bottom-right-radius: 3px; -} -input[type="submit"]:hover { - color: #ffffff; - background-color: #8ecc4c; -} - -div.sphinxsidebar p.searchtip { - clear: both; - padding: 0.5em 0 0 0; - background: #ddd; - color: #666; - font-size: 0.9em; -} - -/* Sidebar links are unusual */ -div.sphinxsidebar li a, -div.sphinxsidebar p a { - background: #e8ecef; /* In case links overlap main content */ - border-radius: 3px; - -moz-border-radius: 3px; - -webkit-border-radius: 3px; - border: 1px solid transparent; /* To prevent things jumping around on hover */ - padding: 0 5px 0 5px; -} -div.sphinxsidebar li a:hover, -div.sphinxsidebar p a:hover { - color: #111; - text-decoration: none; - border: 1px solid #888; -} -div.sphinxsidebar p.logo a { - border: 0; -} - -/* Tweak any link appearing in a heading */ -div.sphinxsidebar h3 a { -} - - - - -/* OTHER STUFF ------------------------------------------------------------ */ - -cite, code, tt { - font-family: 'Consolas', 'Deja Vu Sans Mono', - 'Bitstream Vera Sans Mono', monospace; - font-size: 0.95em; - letter-spacing: 0.01em; -} - -tt { - background-color: #f2f2f2; - color: #444; -} - -tt.descname, tt.descclassname, tt.xref { - border: 0; -} - -hr { - border: 1px solid #abc; - margin: 2em; -} - - -pre, #_fontwidthtest { - font-family: 'Consolas', 'Deja Vu Sans Mono', - 'Bitstream Vera Sans Mono', monospace; - margin: 1em 2em; - font-size: 0.95em; - letter-spacing: 0.015em; - line-height: 120%; - padding: 0.5em; - border: 1px solid #ccc; - background-color: #eee; - border-radius: 6px; - -moz-border-radius: 6px; - -webkit-border-radius: 6px; -} - -pre a { - color: inherit; - text-decoration: underline; -} - -td.linenos pre { - margin: 1em 0em; -} - -td.code pre { - margin: 1em 0em; -} - -div.quotebar { - background-color: #f8f8f8; - max-width: 250px; - float: right; - padding: 2px 7px; - border: 1px solid #ccc; -} - -div.topic { - background-color: #f8f8f8; -} - -table { - border-collapse: collapse; - margin: 0 -0.5em 0 -0.5em; -} - -table td, table th { - padding: 0.2em 0.5em 0.2em 0.5em; -} - - -/* ADMONITIONS AND WARNINGS ------------------------------------------------- */ - -/* Shared by admonitions, warnings and sidebars */ -div.admonition, -div.warning, -div.sidebar { - font-size: 0.9em; - margin: 2em; - padding: 0; - /* - border-radius: 6px; - -moz-border-radius: 6px; - -webkit-border-radius: 6px; - */ -} -div.admonition p, -div.warning p, -div.sidebar p { - margin: 0.5em 1em 0.5em 1em; - padding: 0; -} -div.admonition pre, -div.warning pre, -div.sidebar pre { - margin: 0.4em 1em 0.4em 1em; -} -div.admonition p.admonition-title, -div.warning p.admonition-title, -div.sidebar p.sidebar-title { - margin: 0; - padding: 0.1em 0 0.1em 0.5em; - color: white; - font-weight: bold; - font-size: 1.1em; - text-shadow: 0 1px rgba(0, 0, 0, 0.5); -} -div.admonition ul, div.admonition ol, -div.warning ul, div.warning ol, -div.sidebar ul, div.sidebar ol { - margin: 0.1em 0.5em 0.5em 3em; - padding: 0; -} - - -/* Admonitions and sidebars only */ -div.admonition, div.sidebar { - border: 1px solid #609060; - background-color: #e9ffe9; -} -div.admonition p.admonition-title, -div.sidebar p.sidebar-title { - background-color: #70A070; - border-bottom: 1px solid #609060; -} - - -/* Warnings only */ -div.warning { - border: 1px solid #900000; - background-color: #ffe9e9; -} -div.warning p.admonition-title { - background-color: #b04040; - border-bottom: 1px solid #900000; -} - - -/* Sidebars only */ -div.sidebar { - max-width: 30%; -} - - - -div.versioninfo { - margin: 1em 0 0 0; - border: 1px solid #ccc; - background-color: #DDEAF0; - padding: 8px; - line-height: 1.3em; - font-size: 0.9em; -} - -.viewcode-back { - font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', - 'Verdana', sans-serif; -} - -div.viewcode-block:target { - background-color: #f4debf; - border-top: 1px solid #ac9; - border-bottom: 1px solid #ac9; -} - -dl { - margin: 1em 0 2.5em 0; -} - -dl dt { - font-style: italic; -} - -dl dd { - color: rgb(68, 68, 68); - font-size: 0.95em; -} - -/* Highlight target when you click an internal link */ -dt:target { - background: #ffe080; -} -/* Don't highlight whole divs */ -div.highlight { - background: transparent; -} -/* But do highlight spans (so search results can be highlighted) */ -span.highlight { - background: #ffe080; -} - -div.footer { - background-color: #465158; - color: #eeeeee; - padding: 0 2em 2em 2em; - clear: both; - font-size: 0.8em; - text-align: center; -} - -p { - margin: 0.8em 0 0.5em 0; -} - -.section p img.math { - margin: 0; -} - - -.section p img { - margin: 1em 2em; -} - - -/* MOBILE LAYOUT -------------------------------------------------------------- */ - -@media screen and (max-width: 600px) { - - h1, h2, h3, h4, h5 { - position: relative; - } - - ul { - padding-left: 1.25em; - } - - div.bodywrapper a.headerlink, #indices-and-tables h1 a { - color: #e6e6e6; - font-size: 80%; - float: right; - line-height: 1.8; - position: absolute; - right: -0.7em; - visibility: inherit; - } - - div.bodywrapper h1 a.headerlink, #indices-and-tables h1 a { - line-height: 1.5; - } - - pre { - font-size: 0.7em; - overflow: auto; - word-wrap: break-word; - white-space: pre-wrap; - } - - div.related ul { - height: 2.5em; - padding: 0; - text-align: left; - } - - div.related ul li { - clear: both; - color: #465158; - padding: 0.2em 0; - } - - div.related ul li:last-child { - border-bottom: 1px dotted #8ca1af; - padding-bottom: 0.4em; - margin-bottom: 1em; - width: 100%; - } - - div.related ul li a { - color: #465158; - padding-right: 0; - } - - div.related ul li a:hover { - background: inherit; - color: inherit; - } - - div.related ul li.right { - clear: none; - padding: 0.65em 0; - margin-bottom: 0.5em; - } - - div.related ul li.right a { - color: #fff; - padding-right: 0.8em; - } - - div.related ul li.right a:hover { - background-color: #8ca1af; - } - - div.body { - clear: both; - min-width: 0; - word-wrap: break-word; - } - - div.bodywrapper { - margin: 0 0 0 0; - } - - div.sphinxsidebar { - float: none; - margin: 0; - width: auto; - } - - div.sphinxsidebar input[type="text"] { - height: 2em; - line-height: 2em; - width: 70%; - } - - div.sphinxsidebar input[type="submit"] { - height: 2em; - margin-left: 0.5em; - width: 20%; - } - - div.sphinxsidebar p.searchtip { - background: inherit; - margin-bottom: 1em; - } - - div.sphinxsidebar ul li, div.sphinxsidebar p.topless { - white-space: normal; - } - - .bodywrapper img { - display: block; - margin-left: auto; - margin-right: auto; - max-width: 100%; - } - - div.documentwrapper { - float: none; - } - - div.admonition, div.warning, pre, blockquote { - margin-left: 0em; - margin-right: 0em; - } - - .body p img { - margin: 0; - } - - #searchbox { - background: transparent; - } - - .related:not(:first-child) li { - display: none; - } - - .related:not(:first-child) li.right { - display: block; - } - - div.footer { - padding: 1em; - } - - .rtd_doc_footer .rtd-badge { - float: none; - margin: 1em auto; - position: static; - } - - .rtd_doc_footer .rtd-badge.revsys-inline { - margin-right: auto; - margin-bottom: 2em; - } - - table.indextable { - display: block; - width: auto; - } - - .indextable tr { - display: block; - } - - .indextable td { - display: block; - padding: 0; - width: auto !important; - } - - .indextable td dt { - margin: 1em 0; - } - - ul.search { - margin-left: 0.25em; - } - - ul.search li div.context { - font-size: 90%; - line-height: 1.1; - margin-bottom: 1; - margin-left: 0; - } -} - - -#version_menu, .rtd-badge.rtd { - -webkit-transition: all 0.25s 0.75s; - transition: all 0.25s 0.75s; -} -.footer_popout:hover #version_menu, .footer_popout:hover .rtd-badge.rtd { - -webkit-transition: all 0.25s 0s; - transition: all 0.25s 0s; -} -.rtd-badge { - position: fixed; - display: block; - bottom: 5px; - height: 40px; - text-indent: -9999em; - border-radius: 3px; - -moz-border-radius: 3px; - -webkit-border-radius: 3px; - box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 1px 0 rgba(255, 255, 255, 0.2) inset; - -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 1px 0 rgba(255, 255, 255, 0.2) inset; - -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 1px 0 rgba(255, 255, 255, 0.2) inset; -} -#version_menu { - position: fixed; - visibility: hidden; - opacity: 0; - bottom: 11px; - right: 47px; - list-style-type: none; - margin: 0; -} -.footer_popout:hover #version_menu { - visibility: visible; - opacity: 1; - right: 166px; -} -#version_menu li { - display: block; - float: right; -} -#version_menu li a { - display: block; - padding: 6px 10px 4px 10px; - margin: 7px 7px 0 0; - font-weight: bold; - font-size: 14px; - height: 20px; - line-height: 17px; - text-decoration: none; - color: #fff; - background: #8ca1af url(http://media.readthedocs.org/images/gradient-light.png) bottom left repeat-x; - border-radius: 3px; - -moz-border-radius: 3px; - -webkit-border-radius: 3px; - box-shadow: 0 1px 1px #465158; - -moz-box-shadow: 0 1px 1px #465158; - -webkit-box-shadow: 0 1px 1px #465158; - text-shadow: 0 1px 1px rgba(0, 0, 0, 0.5); -} - -#version_menu li a:hover { - text-decoration: none; - background-color: #697983; - box-shadow: 0 1px 0px #465158; - -moz-box-shadow: 0 1px 0px #465158; - -webkit-box-shadow: 0 1px 0px #465158; -} - -.rtd-badge.rtd { - background: #3b4449 url(http://media.readthedocs.org/images/badge-rtd.png) scroll top left no-repeat; - border: 1px solid #282E32; - width: 41px; - right: 5px; -} - -.footer_popout:hover .rtd-badge.rtd { - width: 160px; -} - -.rtd-badge.revsys { - display: none; -} - -.rtd-badge.revsys-inline-sponsored { - display none; -} - -.rtd-badge.revsys-inline { - display: none; -} - -.rtd_doc_footer { background-color: #465158;} diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/theme.conf stb-tester-31/vendor/py-lmdb/docs/themes/acid/theme.conf --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/docs/themes/acid/theme.conf 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/docs/themes/acid/theme.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -[theme] -inherit = basic -stylesheet = acid.css -pygments_style = sphinx - -[options] -github_repo = - -rightsidebar = false -stickysidebar = false -collapsiblesidebar = false -externalrefs = false - -footerbgcolor = #11303d -footertextcolor = #ffffff -sidebarbgcolor = #1c4e63 -sidebarbtncolor = #3c6e83 -sidebartextcolor = #ffffff -sidebarlinkcolor = #98dbcc -relbarbgcolor = #133f52 -relbartextcolor = #ffffff -relbarlinkcolor = #ffffff -bgcolor = #ffffff -textcolor = #000000 -headbgcolor = #f2f2f2 -headtextcolor = #20435c -headlinkcolor = #c60f0f -linkcolor = #355f7c -visitedlinkcolor = #355f7c -codebgcolor = #eeffcc -codetextcolor = #333333 - -bodyfont = sans-serif -headfont = 'Trebuchet MS', sans-serif diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/dirtybench-gdbm.py stb-tester-31/vendor/py-lmdb/examples/dirtybench-gdbm.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/dirtybench-gdbm.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/examples/dirtybench-gdbm.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,162 +0,0 @@ - -from pprint import pprint -import os -import shutil -import tempfile - -from time import time as now -import random -import gdbm - -MAP_SIZE = 1048576 * 400 -DB_PATH = '/ram/testdb-gdbm' - -if os.path.exists('/ram'): - DB_PATH = '/ram/testdb-gdbm' -else: - DB_PATH = tempfile.mktemp(prefix='dirtybench-gdbm') - - -def x(): - big = '' # '*' * 400 - - if os.path.exists(DB_PATH): - os.unlink(DB_PATH) - - t0 = now() - words = set(file('/usr/share/dict/words').readlines()) - words.update([w.upper() for w in words]) - words.update([w[::-1] for w in words]) - words.update([w[::-1].upper() for w in words]) - words.update(['-'.join(w) for w in words]) - #words.update(['+'.join(w) for w in words]) - #words.update(['/'.join(w) for w in words]) - words = list(words) - alllen = sum(len(w) for w in words) - avglen = alllen / len(words) - print 'permutate %d words avglen %d took %.2fsec' % (len(words), avglen, now()-t0) - - getword = iter(words).next - - env = gdbm.open(DB_PATH, 'c') - - run = True - t0 = now() - last = t0 - while run: - try: - for _ in xrange(50000): - word = getword() - env[word] = big or word - except StopIteration: - run = False - - t1 = now() - if (t1 - last) > 2: - print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0)) - last = t1 - - t1 = now() - print 'done all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0)) - last = t1 - - print - print - - t0 = now() - lst = sum(env[k] and 1 for k in env.keys()) - t1 = now() - print 'enum %d (key, value) pairs took %.2f sec' % ((lst), t1-t0) - - t0 = now() - lst = sum(1 or env[k] for k in reversed(env.keys())) - t1 = now() - print 'reverse enum %d (key, value) pairs took %.2f sec' % ((lst), t1-t0) - - t0 = now() - for word in words: - env[word] - t1 = now() - print 'rand lookup all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0)) - - t0 = now() - for word in words: - hash(env[word]) - t1 = now() - print 'per txn rand lookup+hash all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0)) - - t0 = now() - for word in words: - hash(env[word]) - t1 = now() - print 'rand lookup+hash all keys %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0)) - - t0 = now() - for word in words: - env[word] - t1 = now() - print 'rand lookup all buffers %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0)) - - t0 = now() - for word in words: - hash(env[word]) - t1 = now() - print 'rand lookup+hash all buffers %.2f sec (%d/sec)' % (t1-t0, lst/(t1-t0)) - - - # - # get+put - # - - getword = iter(sorted(words)).next - run = True - t0 = now() - last = t0 - while run: - try: - for _ in xrange(50000): - word = getword() - old = env[word] - env[word] = word - except StopIteration: - run = False - - t1 = now() - if (t1 - last) > 2: - print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0)) - last = t1 - - t1 = now() - print 'get+put all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0)) - last = t1 - - - # - # REPLACE - # - - getword = iter(sorted(words)).next - run = True - t0 = now() - last = t0 - while run: - try: - for _ in xrange(50000): - word = getword() - old = env[word] - except StopIteration: - run = False - - t1 = now() - if (t1 - last) > 2: - print '%.2fs (%d/sec)' % (t1-t0, len(words)/(t1-t0)) - last = t1 - - t1 = now() - print 'replace all %d in %.2fs (%d/sec)' % (len(words), t1-t0, len(words)/(t1-t0)) - last = t1 - - - - -x() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/dirtybench.py stb-tester-31/vendor/py-lmdb/examples/dirtybench.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/dirtybench.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/examples/dirtybench.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,299 +0,0 @@ - -from pprint import pprint -import atexit -import gzip -import itertools -import os -import shutil -import sys -import tempfile - -from time import time as now -import random -import lmdb - -MAP_SIZE = 1048576 * 400 -DB_PATH = '/ram/testdb' -USE_SPARSE_FILES = sys.platform != 'darwin' - -if os.path.exists('/ram'): - DB_PATH = '/ram/testdb' -else: - DB_PATH = tempfile.mktemp(prefix='dirtybench') - - -env = None -@atexit.register -def cleanup(): - if env: - env.close() - if os.path.exists(DB_PATH): - shutil.rmtree(DB_PATH) - - -def reopen_env(**kwargs): - if env: - env.close() - if os.path.exists(DB_PATH): - shutil.rmtree(DB_PATH) - return lmdb.open(DB_PATH, map_size=MAP_SIZE, writemap=USE_SPARSE_FILES, **kwargs) - - -def case(title, **params): - def wrapper(func): - t0 = now() - count = func() - t1 = now() - print('%40s: %2.3fs %8d/sec' % (title, t1-t0, count/(t1-t0))) - return func - return wrapper - - -def x(): - big = '' # '*' * 400 - - t0 = now() - words_path = os.path.join(os.path.dirname(__file__), 'words.gz') - words = set(gzip.open(words_path).read().splitlines()) - words.update([w.upper() for w in words]) - words.update([w[::-1] for w in words]) - words.update([w[::-1].upper() for w in words]) - words.update(['-'.join(w) for w in words]) - #words.update(['+'.join(w) for w in words]) - #words.update(['/'.join(w) for w in words]) - words = list(words) - alllen = sum(len(w) for w in words) - avglen = alllen / len(words) - print 'permutate %d words avglen %d took %.2fsec' % (len(words), avglen, now()-t0) - print 'DB_PATH:', DB_PATH - - words_sorted = sorted(words) - items = [(w, big or w) for w in words] - items_sorted = [(w, big or w) for w in words_sorted] - - global env - env = reopen_env() - - @case('insert') - def test(): - with env.begin(write=True) as txn: - for word in words: - txn.put(word, big or word) - return len(words) - - - st = env.stat() - print - print 'stat:', st - print 'k+v size %.2fkb avg %d, on-disk size: %.2fkb avg %d' %\ - ((2*alllen) / 1024., (2*alllen)/len(words), - (st['psize'] * st['leaf_pages']) / 1024., - (st['psize'] * st['leaf_pages']) / len(words)) - print - - - @case('enum (key, value) pairs') - def test(): - with env.begin() as txn: - return sum(1 for _ in txn.cursor()) - - - @case('reverse enum (key, value) pairs') - def test(): - with env.begin() as txn: - return sum(1 for _ in txn.cursor().iterprev()) - - - @case('enum (key, value) buffers') - def test(): - with env.begin(buffers=True) as txn: - return sum(1 for _ in txn.cursor()) - - - print - - - @case('rand lookup') - def test(): - with env.begin() as txn: - for word in words: - txn.get(word) - return len(words) - - - @case('per txn rand lookup') - def test(): - for word in words: - with env.begin() as txn: - txn.get(word) - return len(words) - - - @case('rand lookup+hash') - def test(): - with env.begin() as txn: - for word in words: - hash(txn.get(word)) - return len(words) - - - @case('rand lookup buffers') - def test(): - with env.begin(buffers=True) as txn: - for word in words: - txn.get(word) - return len(words) - - - @case('rand lookup+hash buffers') - def test(): - with env.begin(buffers=True) as txn: - for word in words: - hash(txn.get(word)) - return len(words) - - - @case('rand lookup buffers (cursor)') - def test(): - with env.begin(buffers=True) as txn: - cursget = txn.cursor().get - for word in words: - cursget(word) - return len(words) - - - print - - - @case('get+put') - def test(): - with env.begin(write=True) as txn: - for word in words: - txn.get(word) - txn.put(word, word) - return len(words) - - - @case('replace') - def test(): - with env.begin(write=True) as txn: - for word in words: - txn.replace(word, word) - return len(words) - - - @case('get+put (cursor)') - def test(): - with env.begin(write=True) as txn: - with txn.cursor() as cursor: - for word in words: - cursor.get(word) - cursor.put(word, word) - return len(words) - - - @case('replace (cursor)') - def test(): - with env.begin(write=True) as txn: - with txn.cursor() as cursor: - for word in words: - cursor.replace(word, word) - return len(words) - - - print - - - env = reopen_env() - @case('insert (rand)') - def test(): - with env.begin(write=True) as txn: - for word in words: - txn.put(word, big or word) - return len(words) - - - env = reopen_env() - @case('insert (seq)') - def test(): - with env.begin(write=True) as txn: - for word in words_sorted: - txn.put(word, big or word) - return len(words) - - - env = reopen_env() - @case('insert (rand), reuse cursor') - def test(): - with env.begin(write=True) as txn: - curs = txn.cursor() - for word in words: - curs.put(word, big or word) - return len(words) - env = reopen_env() - - - @case('insert (seq), reuse cursor') - def test(): - with env.begin(write=True) as txn: - curs = txn.cursor() - for word in words_sorted: - curs.put(word, big or word) - return len(words) - - - env = reopen_env() - @case('insert, putmulti') - def test(): - with env.begin(write=True) as txn: - txn.cursor().putmulti(items) - return len(words) - - - env = reopen_env() - @case('insert, putmulti+generator') - def test(): - with env.begin(write=True) as txn: - txn.cursor().putmulti((w, big or w) for w in words) - return len(words) - - - print - - - env = reopen_env() - @case('append') - def test(): - with env.begin(write=True) as txn: - for word in words_sorted: - txn.put(word, big or word, append=True) - return len(words) - - - env = reopen_env() - @case('append, reuse cursor') - def test(): - with env.begin(write=True) as txn: - curs = txn.cursor() - for word in words_sorted: - curs.put(word, big or word, append=True) - return len(words) - - - env = reopen_env() - @case('append+putmulti') - def test(): - with env.begin(write=True) as txn: - txn.cursor().putmulti(items_sorted, append=True) - return len(words) - - - print - st = env.stat() - print 'stat:', st - print 'k+v size %.2fkb avg %d, on-disk size: %.2fkb avg %d' %\ - ((2*alllen) / 1024., (2*alllen)/len(words), - (st['psize'] * st['leaf_pages']) / 1024., - (st['psize'] * st['leaf_pages']) / len(words)) - -x() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/nastybench.py stb-tester-31/vendor/py-lmdb/examples/nastybench.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/nastybench.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/examples/nastybench.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,110 +0,0 @@ - -# Roughly approximates some of Symas microbenchmark. - -from time import time -import random -import shutil -import os -import tempfile - -import lmdb - - -val = ' ' * 100 -MAX_KEYS = int(1e6) - -t0 = time() - -urandom = file('/dev/urandom', 'rb', 1048576).read - -keys = set() -while len(keys) < MAX_KEYS: - for _ in xrange(min(1000, MAX_KEYS - len(keys))): - keys.add(urandom(16)) - -print 'make %d keys in %.2fsec' % (len(keys), time() - t0) -keys = list(keys) - -if os.path.exists('/ram'): - DB_PATH = '/ram/dbtest' -else: - DB_PATH = tempfile.mktemp(prefix='nastybench') - -if os.path.exists(DB_PATH): - shutil.rmtree(DB_PATH) - -env = lmdb.open(DB_PATH, map_size=1048576 * 1024, - metasync=False, sync=False, map_async=True) - -nextkey = iter(keys).next -run = True -while run: - with env.begin(write=True) as txn: - try: - for _ in xrange(10000): - txn.put(nextkey(), val) - except StopIteration: - run = False - -d = time() - t0 -env.sync(True) -print 'insert %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys) / d) - - - -nextkey = iter(keys).next -t0 = time() - -with env.begin() as txn: - try: - while 1: - txn.get(nextkey()) - except StopIteration: - pass - -d = time() - t0 -print 'random lookup %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d) - - -nextkey = iter(keys).next -t0 = time() - -with env.begin(buffers=True) as txn: - try: - while 1: - txn.get(nextkey()) - except StopIteration: - pass - -d = time() - t0 -print 'random lookup %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d) - - -nextkey = iter(keys).next -t0 = time() - -with env.begin(buffers=True) as txn: - try: - while 1: - hash(txn.get(nextkey())) - except StopIteration: - pass - -d = time() - t0 -print 'random lookup+hash %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d) - - - -nextkey = iter(keys).next -t0 = time() - -with env.begin(buffers=True) as txn: - nextrec = txn.cursor().iternext().next - try: - while 1: - nextrec() - except StopIteration: - pass - -d = time() - t0 -print 'seq read %d buffers in %.2fsec (%d/sec)' % (len(keys), d, len(keys)/d) diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/parabench.py stb-tester-31/vendor/py-lmdb/examples/parabench.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/parabench.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/examples/parabench.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,116 +0,0 @@ - -# Roughly approximates some of Symas microbenchmark. - -import multiprocessing -import os -import random -import shutil -import sys -import tempfile -import time - -try: - import affinity -except: - affinity = False -import lmdb - - -USE_SPARSE_FILES = sys.platform != 'darwin' -DB_PATH = '/ram/dbtest' -MAX_KEYS = int(4e6) - -if os.path.exists('/ram'): - DB_PATH = '/ram/dbtest' -else: - DB_PATH = tempfile.mktemp(prefix='parabench') - - -def open_env(): - return lmdb.open(DB_PATH, - map_size=1048576 * 1024, - metasync=False, - sync=False, - map_async=True, - writemap=USE_SPARSE_FILES) - - -def make_keys(): - t0 = time.time() - urandom = file('/dev/urandom', 'rb', 1048576).read - - keys = set() - while len(keys) < MAX_KEYS: - for _ in xrange(min(1000, MAX_KEYS - len(keys))): - keys.add(urandom(16)) - - print 'make %d keys in %.2fsec' % (len(keys), time.time() - t0) - keys = list(keys) - - nextkey = iter(keys).next - run = True - val = ' ' * 100 - env = open_env() - while run: - with env.begin(write=True) as txn: - try: - for _ in xrange(10000): - txn.put(nextkey(), val) - except StopIteration: - run = False - - d = time.time() - t0 - env.sync(True) - env.close() - print 'insert %d keys in %.2fsec (%d/sec)' % (len(keys), d, len(keys) / d) - - -if 'drop' in sys.argv and os.path.exists(DB_PATH): - shutil.rmtree(DB_PATH) - -if not os.path.exists(DB_PATH): - make_keys() - - -env = open_env() -with env.begin() as txn: - keys = list(txn.cursor().iternext(values=False)) -env.close() - - -def run(idx): - if affinity: - affinity.set_process_affinity_mask(os.getpid(), 1 << idx) - - env = open_env() - k = list(keys) - random.shuffle(k) - k = k[:1000] - - while 1: - with env.begin() as txn: - nextkey = iter(k).next - try: - while 1: - hash(txn.get(nextkey())) - except StopIteration: - pass - arr[idx] += len(k) - - - -nproc = int(sys.argv[1]) -arr = multiprocessing.Array('L', xrange(nproc)) -for x in xrange(nproc): - arr[x] = 0 -procs = [multiprocessing.Process(target=run, args=(x,)) for x in xrange(nproc)] -[p.start() for p in procs] - - -t0 = time.time() -while True: - time.sleep(2) - d = time.time() - t0 - lk = sum(arr) - print 'lookup %d keys in %.2fsec (%d/sec)' % (lk, d, lk / d) - Binary files /tmp/tmpAqZNQV/dFicMWihaU/stb-tester-30-5-gbefe47c/vendor/py-lmdb/examples/words.gz and /tmp/tmpAqZNQV/WvNiqLHPcm/stb-tester-31/vendor/py-lmdb/examples/words.gz differ diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/.gitignore stb-tester-31/vendor/py-lmdb/.gitignore --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/.gitignore 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -MANIFEST -__pycache__ -build -dist -docs/_build -ll.sh -lmdb.egg-info -lmdb/_config.py -lo.sh -old -.tox/ diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/lmdb.h stb-tester-31/vendor/py-lmdb/lib/lmdb.h --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/lmdb.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lib/lmdb.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,1556 +0,0 @@ -/** @file lmdb.h - * @brief Lightning memory-mapped database library - * - * @mainpage Lightning Memory-Mapped Database Manager (LMDB) - * - * @section intro_sec Introduction - * LMDB is a Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. The entire database is exposed - * in a memory map, and all data fetches return data directly - * from the mapped memory, so no malloc's or memcpy's occur during - * data fetches. As such, the library is extremely simple because it - * requires no page caching layer of its own, and it is extremely high - * performance and memory-efficient. It is also fully transactional with - * full ACID semantics, and when the memory map is read-only, the - * database integrity cannot be corrupted by stray pointer writes from - * application code. - * - * The library is fully thread-aware and supports concurrent read/write - * access from multiple processes and threads. Data pages use a copy-on- - * write strategy so no active data pages are ever overwritten, which - * also provides resistance to corruption and eliminates the need of any - * special recovery procedures after a system crash. Writes are fully - * serialized; only one write transaction may be active at a time, which - * guarantees that writers can never deadlock. The database structure is - * multi-versioned so readers run with no locks; writers cannot block - * readers, and readers don't block writers. - * - * Unlike other well-known database mechanisms which use either write-ahead - * transaction logs or append-only data writes, LMDB requires no maintenance - * during operation. Both write-ahead loggers and append-only databases - * require periodic checkpointing and/or compaction of their log or database - * files otherwise they grow without bound. LMDB tracks free pages within - * the database and re-uses them for new write operations, so the database - * size does not grow without bound in normal use. - * - * The memory map can be used as a read-only or read-write map. It is - * read-only by default as this provides total immunity to corruption. - * Using read-write mode offers much higher write performance, but adds - * the possibility for stray application writes thru pointers to silently - * corrupt the database. Of course if your application code is known to - * be bug-free (...) then this is not an issue. - * - * @section caveats_sec Caveats - * Troubleshooting the lock file, plus semaphores on BSD systems: - * - * - A broken lockfile can cause sync issues. - * Stale reader transactions left behind by an aborted program - * cause further writes to grow the database quickly, and - * stale locks can block further operation. - * - * Fix: Check for stale readers periodically, using the - * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just - * make all programs using the database close it; the lockfile - * is always reset on first open of the environment. - * - * - On BSD systems or others configured with MDB_USE_POSIX_SEM, - * startup can fail due to semaphores owned by another userid. - * - * Fix: Open and close the database as the user which owns the - * semaphores (likely last user) or as root, while no other - * process is using the database. - * - * Restrictions/caveats (in addition to those listed for some functions): - * - * - Only the database owner should normally use the database on - * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. - * Multiple users can cause startup to fail later, as noted above. - * - * - There is normally no pure read-only mode, since readers need write - * access to locks and lock file. Exceptions: On read-only filesystems - * or with the #MDB_NOLOCK flag described under #mdb_env_open(). - * - * - By default, in versions before 0.9.10, unused portions of the data - * file might receive garbage data from memory freed by other code. - * (This does not happen when using the #MDB_WRITEMAP flag.) As of - * 0.9.10 the default behavior is to initialize such memory before - * writing to the data file. Since there may be a slight performance - * cost due to this initialization, applications may disable it using - * the #MDB_NOMEMINIT flag. Applications handling sensitive data - * which must not be written should not use this flag. This flag is - * irrelevant when using #MDB_WRITEMAP. - * - * - A thread can only use one transaction at a time, plus any child - * transactions. Each transaction belongs to one thread. See below. - * The #MDB_NOTLS flag changes this for read-only transactions. - * - * - Use an MDB_env* in the process which opened it, without fork()ing. - * - * - Do not have open an LMDB database twice in the same process at - * the same time. Not even from a plain open() call - close()ing it - * breaks flock() advisory locking. - * - * - Avoid long-lived transactions. Read transactions prevent - * reuse of pages freed by newer write transactions, thus the - * database can grow quickly. Write transactions prevent - * other write transactions, since writes are serialized. - * - * - Avoid suspending a process with active transactions. These - * would then be "long-lived" as above. Also read transactions - * suspended when writers commit could sometimes see wrong data. - * - * ...when several processes can use a database concurrently: - * - * - Avoid aborting a process with an active transaction. - * The transaction becomes "long-lived" as above until a check - * for stale readers is performed or the lockfile is reset, - * since the process may not remove it from the lockfile. - * - * - If you do that anyway, do a periodic check for stale readers. Or - * close the environment once in a while, so the lockfile can get reset. - * - * - Do not use LMDB databases on remote filesystems, even between - * processes on the same host. This breaks flock() on some OSes, - * possibly memory map sync, and certainly sync between programs - * on different hosts. - * - * - Opening a database can fail if another process is opening or - * closing it at exactly the same time. - * - * @author Howard Chu, Symas Corporation. - * - * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * @par Derived From: - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _LMDB_H_ -#define _LMDB_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Unix permissions for creating files, or dummy definition for Windows */ -#ifdef _MSC_VER -typedef int mdb_mode_t; -#else -typedef mode_t mdb_mode_t; -#endif - -/** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#ifdef _WIN32 -typedef void *mdb_filehandle_t; -#else -typedef int mdb_filehandle_t; -#endif - -/** @defgroup mdb LMDB API - * @{ - * @brief OpenLDAP Lightning Memory-Mapped Database Manager - */ -/** @defgroup Version Version Macros - * @{ - */ -/** Library major version */ -#define MDB_VERSION_MAJOR 0 -/** Library minor version */ -#define MDB_VERSION_MINOR 9 -/** Library patch version */ -#define MDB_VERSION_PATCH 14 - -/** Combine args a,b,c into a single integer for easy version comparisons */ -#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) - -/** The full library version as a single integer */ -#define MDB_VERSION_FULL \ - MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) - -/** The release date of this library version */ -#define MDB_VERSION_DATE "September 20, 2014" - -/** A stringifier for the version info */ -#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" - -/** A helper for the stringifier macro */ -#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) - -/** The full library version as a C string */ -#define MDB_VERSION_STRING \ - MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) -/** @} */ - -/** @brief Opaque structure for a database environment. - * - * A DB environment supports multiple databases, all residing in the same - * shared-memory map. - */ -typedef struct MDB_env MDB_env; - -/** @brief Opaque structure for a transaction handle. - * - * All database operations require a transaction handle. Transactions may be - * read-only or read-write. - */ -typedef struct MDB_txn MDB_txn; - -/** @brief A handle for an individual database in the DB environment. */ -typedef unsigned int MDB_dbi; - -/** @brief Opaque structure for navigating through a database */ -typedef struct MDB_cursor MDB_cursor; - -/** @brief Generic structure used for passing keys and data in and out - * of the database. - * - * Values returned from the database are valid only until a subsequent - * update operation, or the end of the transaction. Do not modify or - * free them, they commonly point into the database itself. - * - * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. - * The same applies to data sizes in databases with the #MDB_DUPSORT flag. - * Other data items can in theory be from 0 to 0xffffffff bytes long. - */ -typedef struct MDB_val { - size_t mv_size; /**< size of the data item */ - void *mv_data; /**< address of the data item */ -} MDB_val; - -/** @brief A callback function used to compare two keys in a database */ -typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); - -/** @brief A callback function used to relocate a position-dependent data item - * in a fixed-address database. - * - * The \b newptr gives the item's desired address in - * the memory map, and \b oldptr gives its previous address. The item's actual - * data resides at the address in \b item. This callback is expected to walk - * through the fields of the record in \b item and modify any - * values based at the \b oldptr address to be relative to the \b newptr address. - * @param[in,out] item The item that is to be relocated. - * @param[in] oldptr The previous address. - * @param[in] newptr The new address to relocate to. - * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). - * @todo This feature is currently unimplemented. - */ -typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); - -/** @defgroup mdb_env Environment Flags - * @{ - */ - /** mmap at a fixed address (experimental) */ -#define MDB_FIXEDMAP 0x01 - /** no environment directory */ -#define MDB_NOSUBDIR 0x4000 - /** don't fsync after commit */ -#define MDB_NOSYNC 0x10000 - /** read only */ -#define MDB_RDONLY 0x20000 - /** don't fsync metapage after commit */ -#define MDB_NOMETASYNC 0x40000 - /** use writable mmap */ -#define MDB_WRITEMAP 0x80000 - /** use asynchronous msync when #MDB_WRITEMAP is used */ -#define MDB_MAPASYNC 0x100000 - /** tie reader locktable slots to #MDB_txn objects instead of to threads */ -#define MDB_NOTLS 0x200000 - /** don't do any locking, caller must manage their own locks */ -#define MDB_NOLOCK 0x400000 - /** don't do readahead (no effect on Windows) */ -#define MDB_NORDAHEAD 0x800000 - /** don't initialize malloc'd memory before writing to datafile */ -#define MDB_NOMEMINIT 0x1000000 -/** @} */ - -/** @defgroup mdb_dbi_open Database Flags - * @{ - */ - /** use reverse string keys */ -#define MDB_REVERSEKEY 0x02 - /** use sorted duplicates */ -#define MDB_DUPSORT 0x04 - /** numeric keys in native byte order. - * The keys must all be of the same size. */ -#define MDB_INTEGERKEY 0x08 - /** with #MDB_DUPSORT, sorted dup items have fixed size */ -#define MDB_DUPFIXED 0x10 - /** with #MDB_DUPSORT, dups are numeric in native byte order */ -#define MDB_INTEGERDUP 0x20 - /** with #MDB_DUPSORT, use reverse string dups */ -#define MDB_REVERSEDUP 0x40 - /** create DB if not already existing */ -#define MDB_CREATE 0x40000 -/** @} */ - -/** @defgroup mdb_put Write Flags - * @{ - */ -/** For put: Don't write if the key already exists. */ -#define MDB_NOOVERWRITE 0x10 -/** Only for #MDB_DUPSORT
- * For put: don't write if the key and data pair already exist.
- * For mdb_cursor_del: remove all duplicate data items. - */ -#define MDB_NODUPDATA 0x20 -/** For mdb_cursor_put: overwrite the current key/data pair */ -#define MDB_CURRENT 0x40 -/** For put: Just reserve space for data, don't copy it. Return a - * pointer to the reserved space. - */ -#define MDB_RESERVE 0x10000 -/** Data is being appended, don't split full pages. */ -#define MDB_APPEND 0x20000 -/** Duplicate data is being appended, don't split full pages. */ -#define MDB_APPENDDUP 0x40000 -/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ -#define MDB_MULTIPLE 0x80000 -/* @} */ - -/** @defgroup mdb_copy Copy Flags - * @{ - */ -/** Compacting copy: Omit free space from copy, and renumber all - * pages sequentially. - */ -#define MDB_CP_COMPACT 0x01 -/* @} */ - -/** @brief Cursor Get operations. - * - * This is the set of all operations for retrieving data - * using a cursor. - */ -typedef enum MDB_cursor_op { - MDB_FIRST, /**< Position at first key/data item */ - MDB_FIRST_DUP, /**< Position at first data item of current key. - Only for #MDB_DUPSORT */ - MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ - MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ - MDB_GET_CURRENT, /**< Return key/data at current cursor position */ - MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items - from current cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_LAST, /**< Position at last key/data item */ - MDB_LAST_DUP, /**< Position at last data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT, /**< Position at next data item */ - MDB_NEXT_DUP, /**< Position at next data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items - from next cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_NEXT_NODUP, /**< Position at first data item of next key */ - MDB_PREV, /**< Position at previous data item */ - MDB_PREV_DUP, /**< Position at previous data item of current key. - Only for #MDB_DUPSORT */ - MDB_PREV_NODUP, /**< Position at last data item of previous key */ - MDB_SET, /**< Position at specified key */ - MDB_SET_KEY, /**< Position at specified key, return key + data */ - MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ -} MDB_cursor_op; - -/** @defgroup errors Return Codes - * - * BerkeleyDB uses -30800 to -30999, we'll go under them - * @{ - */ - /** Successful result */ -#define MDB_SUCCESS 0 - /** key/data pair already exists */ -#define MDB_KEYEXIST (-30799) - /** key/data pair not found (EOF) */ -#define MDB_NOTFOUND (-30798) - /** Requested page not found - this usually indicates corruption */ -#define MDB_PAGE_NOTFOUND (-30797) - /** Located page was wrong type */ -#define MDB_CORRUPTED (-30796) - /** Update of meta page failed, probably I/O error */ -#define MDB_PANIC (-30795) - /** Environment version mismatch */ -#define MDB_VERSION_MISMATCH (-30794) - /** File is not a valid LMDB file */ -#define MDB_INVALID (-30793) - /** Environment mapsize reached */ -#define MDB_MAP_FULL (-30792) - /** Environment maxdbs reached */ -#define MDB_DBS_FULL (-30791) - /** Environment maxreaders reached */ -#define MDB_READERS_FULL (-30790) - /** Too many TLS keys in use - Windows only */ -#define MDB_TLS_FULL (-30789) - /** Txn has too many dirty pages */ -#define MDB_TXN_FULL (-30788) - /** Cursor stack too deep - internal error */ -#define MDB_CURSOR_FULL (-30787) - /** Page has not enough space - internal error */ -#define MDB_PAGE_FULL (-30786) - /** Database contents grew beyond environment mapsize */ -#define MDB_MAP_RESIZED (-30785) - /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ -#define MDB_INCOMPATIBLE (-30784) - /** Invalid reuse of reader locktable slot */ -#define MDB_BAD_RSLOT (-30783) - /** Transaction cannot recover - it must be aborted */ -#define MDB_BAD_TXN (-30782) - /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ -#define MDB_BAD_VALSIZE (-30781) - /** The specified DBI was changed unexpectedly */ -#define MDB_BAD_DBI (-30780) - /** The last defined error code */ -#define MDB_LAST_ERRCODE MDB_BAD_DBI -/** @} */ - -/** @brief Statistics for a database in the environment */ -typedef struct MDB_stat { - unsigned int ms_psize; /**< Size of a database page. - This is currently the same for all databases. */ - unsigned int ms_depth; /**< Depth (height) of the B-tree */ - size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ - size_t ms_leaf_pages; /**< Number of leaf pages */ - size_t ms_overflow_pages; /**< Number of overflow pages */ - size_t ms_entries; /**< Number of data items */ -} MDB_stat; - -/** @brief Information about the environment */ -typedef struct MDB_envinfo { - void *me_mapaddr; /**< Address of map, if fixed */ - size_t me_mapsize; /**< Size of the data memory map */ - size_t me_last_pgno; /**< ID of the last used page */ - size_t me_last_txnid; /**< ID of the last committed transaction */ - unsigned int me_maxreaders; /**< max reader slots in the environment */ - unsigned int me_numreaders; /**< max reader slots used in the environment */ -} MDB_envinfo; - - /** @brief Return the LMDB library version information. - * - * @param[out] major if non-NULL, the library major version number is copied here - * @param[out] minor if non-NULL, the library minor version number is copied here - * @param[out] patch if non-NULL, the library patch version number is copied here - * @retval "version string" The library version as a string - */ -char *mdb_version(int *major, int *minor, int *patch); - - /** @brief Return a string describing a given error code. - * - * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) - * function. If the error code is greater than or equal to 0, then the string - * returned by the system function strerror(3) is returned. If the error code - * is less than 0, an error string corresponding to the LMDB library error is - * returned. See @ref errors for a list of LMDB-specific error codes. - * @param[in] err The error code - * @retval "error message" The description of the error - */ -char *mdb_strerror(int err); - - /** @brief Create an LMDB environment handle. - * - * This function allocates memory for a #MDB_env structure. To release - * the allocated memory and discard the handle, call #mdb_env_close(). - * Before the handle may be used, it must be opened using #mdb_env_open(). - * Various other options may also need to be set before opening the handle, - * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), - * depending on usage requirements. - * @param[out] env The address where the new handle will be stored - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_create(MDB_env **env); - - /** @brief Open an environment handle. - * - * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] path The directory in which the database files reside. This - * directory must already exist and be writable. - * @param[in] flags Special options for this environment. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - * Flags set by mdb_env_set_flags() are also used. - *

#MDB_FIXEDMAP - * use a fixed address for the mmap region. This flag must be specified - * when creating the environment, and is stored persistently in the environment. - * If successful, the memory map will always reside at the same virtual address - * and pointers used to reference data items in the database will be constant - * across multiple invocations. This option may not always work, depending on - * how the operating system has allocated memory to shared libraries and other uses. - * The feature is highly experimental. - *
#MDB_NOSUBDIR - * By default, LMDB creates its environment in a directory whose - * pathname is given in \b path, and creates its data and lock files - * under that directory. With this option, \b path is used as-is for - * the database main data file. The database lock file is the \b path - * with "-lock" appended. - *
#MDB_RDONLY - * Open the environment in read-only mode. No write operations will be - * allowed. LMDB will still modify the lock file - except on read-only - * filesystems, where LMDB does not use locks. - *
#MDB_WRITEMAP - * Use a writeable memory map unless MDB_RDONLY is set. This is faster - * and uses fewer mallocs, but loses protection from application bugs - * like wild pointer writes and other bad updates into the database. - * Incompatible with nested transactions. - * Do not mix processes with and without MDB_WRITEMAP on the same - * environment. This can defeat durability (#mdb_env_sync etc). - *
#MDB_NOMETASYNC - * Flush system buffers to disk only once per transaction, omit the - * metadata flush. Defer that until the system flushes files to disk, - * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization - * maintains database integrity, but a system crash may undo the last - * committed transaction. I.e. it preserves the ACI (atomicity, - * consistency, isolation) but not D (durability) database property. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
#MDB_NOSYNC - * Don't flush system buffers to disk when committing a transaction. - * This optimization means a system crash can corrupt the database or - * lose the last transactions if buffers are not yet flushed to disk. - * The risk is governed by how often the system flushes dirty buffers - * to disk and how often #mdb_env_sync() is called. However, if the - * filesystem preserves write order and the #MDB_WRITEMAP flag is not - * used, transactions exhibit ACI (atomicity, consistency, isolation) - * properties and only lose D (durability). I.e. database integrity - * is maintained, but a system crash may undo the final transactions. - * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no - * hint for when to write transactions to disk, unless #mdb_env_sync() - * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
#MDB_MAPASYNC - * When using #MDB_WRITEMAP, use asynchronous flushes to disk. - * As with #MDB_NOSYNC, a system crash can then corrupt the - * database or lose the last transactions. Calling #mdb_env_sync() - * ensures on-disk database integrity until next commit. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
#MDB_NOTLS - * Don't use Thread-Local Storage. Tie reader locktable slots to - * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps - * the slot reseved for the #MDB_txn object. A thread may use parallel - * read-only transactions. A read-only transaction may span threads if - * the user synchronizes its use. Applications that multiplex many - * user threads over individual OS threads need this option. Such an - * application must also serialize the write transactions in an OS - * thread, since LMDB's write locking is unaware of the user threads. - *
#MDB_NOLOCK - * Don't do any locking. If concurrent access is anticipated, the - * caller must manage all concurrency itself. For proper operation - * the caller must enforce single-writer semantics, and must ensure - * that no readers are using old transactions while a writer is - * active. The simplest approach is to use an exclusive lock so that - * no readers may be active at all when a writer begins. - *
#MDB_NORDAHEAD - * Turn off readahead. Most operating systems perform readahead on - * read requests by default. This option turns it off if the OS - * supports it. Turning it off may help random read performance - * when the DB is larger than RAM and system RAM is full. - * The option is not implemented on Windows. - *
#MDB_NOMEMINIT - * Don't initialize malloc'd memory before writing to unused spaces - * in the data file. By default, memory for pages written to the data - * file is obtained using malloc. While these pages may be reused in - * subsequent transactions, freshly malloc'd pages will be initialized - * to zeroes before use. This avoids persisting leftover data from other - * code (that used the heap and subsequently freed the memory) into the - * data file. Note that many other system libraries may allocate - * and free memory from the heap for arbitrary uses. E.g., stdio may - * use the heap for file I/O buffers. This initialization step has a - * modest performance cost so some applications may want to disable - * it using this flag. This option can be a problem for applications - * which handle sensitive data like passwords, and it makes memory - * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, - * which writes directly to the mmap instead of using malloc for pages. The - * initialization is also skipped if #MDB_RESERVE is used; the - * caller is expected to overwrite all of the memory that was - * reserved in that case. - * This flag may be changed at any time using #mdb_env_set_flags(). - *

- * @param[in] mode The UNIX permissions to set on created files. This parameter - * is ignored on Windows. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the - * version that created the database environment. - *
#MDB_INVALID - the environment file headers are corrupted. - *
ENOENT - the directory specified by the path parameter doesn't exist. - *
EACCES - the user didn't have permission to access the environment files. - *
EAGAIN - the environment was locked by another process. - *

- */ -int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); - - /** @brief Copy an LMDB environment to the specified path. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy(MDB_env *env, const char *path); - - /** @brief Copy an LMDB environment to the specified file descriptor. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); - - /** @brief Copy an LMDB environment to the specified path, with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *

#MDB_CP_COMPACT - Perform compaction while copying: omit free - * pages and sequentially renumber all pages in output. This option - * consumes more CPU and runs more slowly than the default. - *

- * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); - - /** @brief Copy an LMDB environment to the specified file descriptor, - * with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. See - * #mdb_env_copy2() for further details. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @param[in] flags Special options for this operation. - * See #mdb_env_copy2() for options. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); - - /** @brief Return statistics about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - */ -int mdb_env_stat(MDB_env *env, MDB_stat *stat); - - /** @brief Return information about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_envinfo structure - * where the information will be copied - */ -int mdb_env_info(MDB_env *env, MDB_envinfo *stat); - - /** @brief Flush the data buffers to disk. - * - * Data is always written to disk when #mdb_txn_commit() is called, - * but the operating system may keep it buffered. LMDB always flushes - * the OS buffers upon commit as well, unless the environment was - * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is - * not valid if the environment was opened with #MDB_RDONLY. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] force If non-zero, force a synchronous flush. Otherwise - * if the environment has the #MDB_NOSYNC flag set the flushes - * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EACCES - the environment is read-only. - *
EINVAL - an invalid parameter was specified. - *
EIO - an error occurred during synchronization. - *

- */ -int mdb_env_sync(MDB_env *env, int force); - - /** @brief Close the environment and release the memory map. - * - * Only a single thread may call this function. All transactions, databases, - * and cursors must already be closed before calling this function. Attempts to - * use any such handles after calling this function will cause a SIGSEGV. - * The environment handle will be freed and must not be used again after this call. - * @param[in] env An environment handle returned by #mdb_env_create() - */ -void mdb_env_close(MDB_env *env); - - /** @brief Set environment flags. - * - * This may be used to set some flags in addition to those from - * #mdb_env_open(), or to unset these flags. If several threads - * change the flags at the same time, the result is undefined. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] flags The flags to change, bitwise OR'ed together - * @param[in] onoff A non-zero value sets the flags, zero clears them. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); - - /** @brief Get environment flags. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] flags The address of an integer to store the flags - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_env_get_flags(MDB_env *env, unsigned int *flags); - - /** @brief Return the path that was used in #mdb_env_open(). - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] path Address of a string pointer to contain the path. This - * is the actual string in the environment, not a copy. It should not be - * altered in any way. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_env_get_path(MDB_env *env, const char **path); - - /** @brief Return the filedescriptor for the given environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); - - /** @brief Set the size of the memory map to use for this environment. - * - * The size should be a multiple of the OS page size. The default is - * 10485760 bytes. The size of the memory map is also the maximum size - * of the database. The value should be chosen as large as possible, - * to accommodate future growth of the database. - * This function should be called after #mdb_env_create() and before #mdb_env_open(). - * It may be called at later times if no transactions are active in - * this process. Note that the library does not check for this condition, - * the caller must ensure it explicitly. - * - * The new size takes effect immediately for the current process but - * will not be persisted to any others until a write transaction has been - * committed by the current process. Also, only mapsize increases are - * persisted into the environment. - * - * If the mapsize is increased by another process, and data has grown - * beyond the range of the current mapsize, #mdb_txn_begin() will - * return #MDB_MAP_RESIZED. This function may be called with a size - * of zero to adopt the new size. - * - * Any attempt to set a size smaller than the space already consumed - * by the environment will be silently changed to the current size of the used space. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] size The size in bytes - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified, or the environment has - * an active write transaction. - *

- */ -int mdb_env_set_mapsize(MDB_env *env, size_t size); - - /** @brief Set the maximum number of threads/reader slots for the environment. - * - * This defines the number of slots in the lock table that is used to track readers in the - * the environment. The default is 126. - * Starting a read-only transaction normally ties a lock table slot to the - * current thread until the environment closes or the thread exits. If - * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the - * MDB_txn object until it or the #MDB_env object is destroyed. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] readers The maximum number of reader lock table slots - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified, or the environment is already open. - *

- */ -int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); - - /** @brief Get the maximum number of threads/reader slots for the environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] readers Address of an integer to store the number of readers - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); - - /** @brief Set the maximum number of named databases for the environment. - * - * This function is only needed if multiple databases will be used in the - * environment. Simpler applications that use the environment as a single - * unnamed database can ignore this option. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * - * Currently a moderate number of slots are cheap but a huge number gets - * expensive: 7-120 words per transaction, and every #mdb_dbi_open() - * does a linear search of the opened slots. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbs The maximum number of databases - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified, or the environment is already open. - *

- */ -int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); - - /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. - * - * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. - * See @ref MDB_val. - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The maximum size of a key we can write - */ -int mdb_env_get_maxkeysize(MDB_env *env); - - /** @brief Set application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_userctx(MDB_env *env, void *ctx); - - /** @brief Get the application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The pointer set by #mdb_env_set_userctx(). - */ -void *mdb_env_get_userctx(MDB_env *env); - - /** @brief A callback function for most LMDB assert() failures, - * called before printing the message and aborting. - * - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] msg The assertion message, not including newline. - */ -typedef void MDB_assert_func(MDB_env *env, const char *msg); - - /** Set or reset the assert() callback of the environment. - * Disabled if liblmdb is buillt with NDEBUG. - * @note This hack should become obsolete as lmdb's error handling matures. - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] func An #MDB_assert_func function, or 0. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); - - /** @brief Create a transaction for use with the environment. - * - * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). - * @note A transaction and its cursors must only be used by a single - * thread, and a thread may only have a single transaction at a time. - * If #MDB_NOTLS is in use, this does not apply to read-only transactions. - * @note Cursors may not span transactions. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] parent If this parameter is non-NULL, the new transaction - * will be a nested transaction, with the transaction indicated by \b parent - * as its parent. Transactions may be nested to any level. A parent - * transaction and its cursors may not issue any other operations than - * mdb_txn_commit and mdb_txn_abort while it has active child transactions. - * @param[in] flags Special options for this transaction. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *

#MDB_RDONLY - * This transaction will not perform any write operations. - *

- * @param[out] txn Address where the new #MDB_txn handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's - * mapsize and this environment's map must be resized as well. - * See #mdb_env_set_mapsize(). - *
#MDB_READERS_FULL - a read-only transaction was requested and - * the reader lock table is full. See #mdb_env_set_maxreaders(). - *
ENOMEM - out of memory. - *

- */ -int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); - - /** @brief Returns the transaction's #MDB_env - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -MDB_env *mdb_txn_env(MDB_txn *txn); - - /** @brief Commit all the operations of a transaction into the database. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *
ENOSPC - no more disk space. - *
EIO - a low-level I/O error occurred while writing. - *
ENOMEM - out of memory. - *

- */ -int mdb_txn_commit(MDB_txn *txn); - - /** @brief Abandon all the operations of the transaction instead of saving them. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_abort(MDB_txn *txn); - - /** @brief Reset a read-only transaction. - * - * Abort the transaction like #mdb_txn_abort(), but keep the transaction - * handle. #mdb_txn_renew() may reuse the handle. This saves allocation - * overhead if the process will start a new read-only transaction soon, - * and also locking overhead if #MDB_NOTLS is in use. The reader table - * lock is released, but the table slot stays tied to its thread or - * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free - * its lock table slot if MDB_NOTLS is in use. - * Cursors opened within the transaction must not be used - * again after this call, except with #mdb_cursor_renew(). - * Reader locks generally don't interfere with writers, but they keep old - * versions of database pages allocated. Thus they prevent the old pages - * from being reused when writers commit new data, and so under heavy load - * the database size may grow much more rapidly than otherwise. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_reset(MDB_txn *txn); - - /** @brief Renew a read-only transaction. - * - * This acquires a new reader lock for a transaction handle that had been - * released by #mdb_txn_reset(). It must be called before a reset transaction - * may be used again. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_txn_renew(MDB_txn *txn); - -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) - - /** @brief Open a database in the environment. - * - * A database handle denotes the name and parameters of a database, - * independently of whether such a database exists. - * The database handle may be discarded by calling #mdb_dbi_close(). - * The old database handle is returned if the database was already open. - * The handle may only be closed once. - * - * The database handle will be private to the current transaction until - * the transaction is successfully committed. If the transaction is - * aborted the handle will be closed automatically. - * After a successful commit the handle will reside in the shared - * environment, and may be used by other transactions. - * - * This function must not be called from multiple concurrent - * transactions in the same process. A transaction that uses - * this function must finish (either commit or abort) before - * any other transaction in the process may use this function. - * - * To use named databases (with name != NULL), #mdb_env_set_maxdbs() - * must be called before opening the environment. Database names - * are kept as keys in the unnamed database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] name The name of the database to open. If only a single - * database is needed in the environment, this value may be NULL. - * @param[in] flags Special options for this database. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *

#MDB_REVERSEKEY - * Keys are strings to be compared in reverse order, from the end - * of the strings to the beginning. By default, Keys are treated as strings and - * compared from beginning to end. - *
#MDB_DUPSORT - * Duplicate keys may be used in the database. (Or, from another perspective, - * keys may have multiple data items, stored in sorted order.) By default - * keys must be unique and may have only a single data item. - *
#MDB_INTEGERKEY - * Keys are binary integers in native byte order. Setting this option - * requires all keys to be the same size, typically sizeof(int) - * or sizeof(size_t). - *
#MDB_DUPFIXED - * This flag may only be used in combination with #MDB_DUPSORT. This option - * tells the library that the data items for this database are all the same - * size, which allows further optimizations in storage and retrieval. When - * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE - * cursor operations may be used to retrieve multiple items at once. - *
#MDB_INTEGERDUP - * This option specifies that duplicate data items are also integers, and - * should be sorted as such. - *
#MDB_REVERSEDUP - * This option specifies that duplicate data items should be compared as - * strings in reverse order. - *
#MDB_CREATE - * Create the named database if it doesn't exist. This option is not - * allowed in a read-only transaction or a read-only environment. - *

- * @param[out] dbi Address where the new #MDB_dbi handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_NOTFOUND - the specified database doesn't exist in the environment - * and #MDB_CREATE was not specified. - *
#MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). - *

- */ -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); - - /** @brief Retrieve statistics for a database. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); - - /** @brief Retrieve the DB flags for a database handle. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] flags Address where the flags will be returned. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); - - /** @brief Close a database handle. Normally unnecessary. Use with care: - * - * This call is not mutex protected. Handles should only be closed by - * a single thread, and only if no other threads are going to reference - * the database handle or one of its cursors any further. Do not close - * a handle if an existing transaction has modified its database. - * Doing so can cause misbehavior from database corruption to errors - * like MDB_BAD_VALSIZE (since the DB name is gone). - * - * Closing a database handle is not necessary, but lets #mdb_dbi_open() - * reuse the handle value. Usually it's better to set a bigger - * #mdb_env_set_maxdbs(), unless that value would be large. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); - - /** @brief Empty or delete+close a database. - * - * See #mdb_dbi_close() for restrictions about closing the DB handle. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] del 0 to empty the DB, 1 to delete it from the - * environment and close the DB handle. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); - - /** @brief Set a custom key comparison function for a database. - * - * The comparison function is called whenever it is necessary to compare a - * key specified by the application with a key currently stored in the database. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating - * before longer keys. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. - * - * This comparison function is called whenever it is necessary to compare a data - * item specified by the application with a data item currently stored in the database. - * This function only takes effect if the database was opened with the #MDB_DUPSORT - * flag. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating - * before longer items. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a relocation function for a #MDB_FIXEDMAP database. - * - * @todo The relocation function is called whenever it is necessary to move the data - * of an item to a different position in the database (e.g. through tree - * balancing operations, shifts as a result of adds or deletes, etc.). It is - * intended to allow address/position-dependent data items to be stored in - * a database in an environment opened with the #MDB_FIXEDMAP option. - * Currently the relocation feature is unimplemented and setting - * this function has no effect. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] rel A #MDB_rel_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); - - /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. - * - * See #mdb_set_relfunc and #MDB_rel_func for more details. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * It will be passed to the callback function set by #mdb_set_relfunc - * as its \b relctx parameter whenever the callback is invoked. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); - - /** @brief Get items from a database. - * - * This function retrieves key/data pairs from the database. The address - * and length of the data associated with the specified \b key are returned - * in the structure to which \b data refers. - * If the database supports duplicate keys (#MDB_DUPSORT) then the - * first data item for the key will be returned. Retrieval of other - * items requires the use of #mdb_cursor_get(). - * - * @note The memory pointed to by the returned values is owned by the - * database. The caller need not dispose of the memory, and may not - * modify it in any way. For values returned in a read-only transaction - * any modification attempts will cause a SIGSEGV. - * @note Values returned from the database are valid only until a - * subsequent update operation, or the end of the transaction. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to search for in the database - * @param[out] data The data corresponding to the key - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_NOTFOUND - the key was not in the database. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Store items into a database. - * - * This function stores key/data pairs in the database. The default behavior - * is to enter the new key/data pair, replacing any previously existing key - * if duplicates are disallowed, or adding a duplicate data item if - * duplicates are allowed (#MDB_DUPSORT). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to store in the database - * @param[in,out] data The data to store - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *

#MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
#MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). The \b data - * parameter will be set to point to the existing item. - *
#MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later - before - * the next update operation or the transaction ends. This saves - * an extra memcpy if the data is being generated later. - * LMDB does nothing else with this memory, the caller is expected - * to modify all of the space requested. - *
#MDB_APPEND - append the given key/data pair to the end of the - * database. This option allows fast bulk loading when keys are - * already known to be in the correct order. Loading unsorted keys - * with this flag will cause a #MDB_KEYEXIST error. - *
#MDB_APPENDDUP - as above, but for sorted dup data. - *

- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
#MDB_TXN_FULL - the transaction has too many dirty pages. - *
EACCES - an attempt was made to write in a read-only transaction. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete items from a database. - * - * This function removes key/data pairs from the database. - * If the database does not support sorted duplicate data items - * (#MDB_DUPSORT) the data parameter is ignored. - * If the database supports sorted duplicates and the data parameter - * is NULL, all of the duplicate data items for the key will be - * deleted. Otherwise, if the data parameter is non-NULL - * only the matching data item will be deleted. - * This function will return #MDB_NOTFOUND if the specified key/data - * pair is not in the database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to delete from the database - * @param[in] data The data to delete - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EACCES - an attempt was made to write in a read-only transaction. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Create a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * A cursor cannot be used when its database handle is closed. Nor - * when its transaction has ended, except with #mdb_cursor_renew(). - * It can be discarded with #mdb_cursor_close(). - * A cursor in a write-transaction can be closed before its transaction - * ends, and will otherwise be closed when its transaction ends. - * A cursor in a read-only transaction must be closed explicitly, before - * or after its transaction ends. It can be reused with - * #mdb_cursor_renew() before finally closing it. - * @note Earlier documentation said that cursors in every transaction - * were closed when the transaction committed or aborted. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] cursor Address where the new #MDB_cursor handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); - - /** @brief Close a cursor handle. - * - * The cursor handle will be freed and must not be used again after this call. - * Its transaction must still be live if it is a write-transaction. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -void mdb_cursor_close(MDB_cursor *cursor); - - /** @brief Renew a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * Cursors that are only used in read-only - * transactions may be re-used, to avoid unnecessary malloc/free overhead. - * The cursor may be associated with a new read-only transaction, and - * referencing the same database handle as it was created with. - * This may be done whether the previous transaction is live or dead. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - an invalid parameter was specified. - *

- */ -int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); - - /** @brief Return the cursor's transaction handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); - - /** @brief Return the cursor's database handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); - - /** @brief Retrieve by cursor. - * - * This function retrieves key/data pairs from the database. The address and length - * of the key are returned in the object to which \b key refers (except for the - * case of the #MDB_SET option, in which the \b key object is unchanged), and - * the address and length of the data are returned in the object to which \b data - * refers. - * See #mdb_get() for restrictions on using the output values. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in,out] key The key for a retrieved item - * @param[in,out] data The data of a retrieved item - * @param[in] op A cursor operation #MDB_cursor_op - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_NOTFOUND - no matching key found. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - MDB_cursor_op op); - - /** @brief Store by cursor. - * - * This function stores key/data pairs into the database. - * The cursor is positioned at the new item, or on failure usually near it. - * @note Earlier documentation incorrectly said errors would leave the - * state of the cursor unchanged. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] key The key operated on. - * @param[in] data The data operated on. - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *

#MDB_CURRENT - replace the item at the current cursor position. - * The \b key parameter must still be provided, and must match it. - * If using sorted duplicates (#MDB_DUPSORT) the data item must still - * sort into the same place. This is intended to be used when the - * new data is the same size as the old. Otherwise it will simply - * perform a delete of the old record followed by an insert. - *
#MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
#MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). - *
#MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later. This saves - * an extra memcpy if the data is being generated later. - *
#MDB_APPEND - append the given key/data pair to the end of the - * database. No key comparisons are performed. This option allows - * fast bulk loading when keys are already known to be in the - * correct order. Loading unsorted keys with this flag will cause - * data corruption. - *
#MDB_APPENDDUP - as above, but for sorted dup data. - *
#MDB_MULTIPLE - store multiple contiguous data elements in a - * single request. This flag may only be specified if the database - * was opened with #MDB_DUPFIXED. The \b data argument must be an - * array of two MDB_vals. The mv_size of the first MDB_val must be - * the size of a single data element. The mv_data of the first MDB_val - * must point to the beginning of the array of contiguous data elements. - * The mv_size of the second MDB_val must be the count of the number - * of data elements to store. On return this field will be set to - * the count of the number of elements actually written. The mv_data - * of the second MDB_val is unused. - *

- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
#MDB_TXN_FULL - the transaction has too many dirty pages. - *
EACCES - an attempt was made to write in a read-only transaction. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete current key/data pair - * - * This function deletes the key/data pair to which the cursor refers. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *

#MDB_NODUPDATA - delete all of the data items for the current key. - * This flag may only be specified if the database was opened with #MDB_DUPSORT. - *

- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EACCES - an attempt was made to write in a read-only transaction. - *
EINVAL - an invalid parameter was specified. - *

- */ -int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); - - /** @brief Return count of duplicates for current key. - * - * This call is only valid on databases that support sorted duplicate - * data items #MDB_DUPSORT. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[out] countp Address where the count will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *

EINVAL - cursor is not initialized, or an invalid parameter was specified. - *

- */ -int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two data items were keys in the - * specified database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two items were data items of - * the specified database. The database must have the #MDB_DUPSORT flag. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief A callback function used to print a message from the library. - * - * @param[in] msg The string to be printed. - * @param[in] ctx An arbitrary context pointer for the callback. - * @return < 0 on failure, >= 0 on success. - */ -typedef int (MDB_msg_func)(const char *msg, void *ctx); - - /** @brief Dump the entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] func A #MDB_msg_func function - * @param[in] ctx Anything the message function needs - * @return < 0 on failure, >= 0 on success. - */ -int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); - - /** @brief Check for stale entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] dead Number of stale slots that were cleared - * @return 0 on success, non-zero on failure. - */ -int mdb_reader_check(MDB_env *env, int *dead); -/** @} */ - -#ifdef __cplusplus -} -#endif -/** @page tools LMDB Command Line Tools - The following describes the command line tools that are available for LMDB. - \li \ref mdb_copy_1 - \li \ref mdb_dump_1 - \li \ref mdb_load_1 - \li \ref mdb_stat_1 -*/ - -#endif /* _LMDB_H_ */ diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/mdb.c stb-tester-31/vendor/py-lmdb/lib/mdb.c --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/mdb.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lib/mdb.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,9543 +0,0 @@ -/** @file mdb.c - * @brief Lightning memory-mapped database library - * - * A Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. - */ -/* - * Copyright 2011-2015 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE 1 -#endif -#ifdef _WIN32 -#include -#include -/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it - * as int64 which is wrong. MSVC doesn't define it at all, so just - * don't use it. - */ -#define MDB_PID_T int -#define MDB_THR_T DWORD -#include -#include -#ifdef __GNUC__ -# include -#else -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# define BYTE_ORDER LITTLE_ENDIAN -# ifndef SSIZE_MAX -# define SSIZE_MAX INT_MAX -# endif -#endif -#else -#include -#include -#define MDB_PID_T pid_t -#define MDB_THR_T pthread_t -#include -#include -#include -#ifdef HAVE_SYS_FILE_H -#include -#endif -#include -#endif - -#if defined(__mips) && defined(__linux) -/* MIPS has cache coherency issues, requires explicit cache control */ -#include -extern int cacheflush(char *addr, int nbytes, int cache); -#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) -#else -#define CACHEFLUSH(addr, bytes, cache) -#endif - -#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) -/** fdatasync is broken on ext3/ext4fs on older kernels, see - * description in #mdb_env_open2 comments. You can safely - * define MDB_FDATASYNC_WORKS if this code will only be run - * on kernels 3.6 and newer. - */ -#define BROKEN_FDATASYNC -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__sun) || defined(ANDROID) -/* Most platforms have posix_memalign, older may only have memalign */ -#define HAVE_MEMALIGN 1 -#include -#endif - -#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) -#include -#include /* defines BYTE_ORDER on HPUX and Solaris */ -#endif - -#if defined(__APPLE__) || defined (BSD) -# define MDB_USE_POSIX_SEM 1 -# define MDB_FDATASYNC fsync -#elif defined(ANDROID) -# define MDB_FDATASYNC fsync -#endif - -#ifndef _WIN32 -#include -#ifdef MDB_USE_POSIX_SEM -# define MDB_USE_HASH 1 -#include -#endif -#endif - -#ifdef USE_VALGRIND -#include -#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) -#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) -#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) -#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) -#else -#define VGMEMP_CREATE(h,r,z) -#define VGMEMP_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) -#define VGMEMP_DESTROY(h) -#define VGMEMP_DEFINED(a,s) -#endif - -#ifndef BYTE_ORDER -# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) -/* Solaris just defines one or the other */ -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# ifdef _LITTLE_ENDIAN -# define BYTE_ORDER LITTLE_ENDIAN -# else -# define BYTE_ORDER BIG_ENDIAN -# endif -# else -# define BYTE_ORDER __BYTE_ORDER -# endif -#endif - -#ifndef LITTLE_ENDIAN -#define LITTLE_ENDIAN __LITTLE_ENDIAN -#endif -#ifndef BIG_ENDIAN -#define BIG_ENDIAN __BIG_ENDIAN -#endif - -#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) -#define MISALIGNED_OK 1 -#endif - -#include "lmdb.h" -#include "midl.h" - -#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) -# error "Unknown or unsupported endianness (BYTE_ORDER)" -#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -# error "Two's complement, reasonably sized integer types, please" -#endif - -#ifdef __GNUC__ -/** Put infrequently used env functions in separate section */ -# ifdef __APPLE__ -# define ESECT __attribute__ ((section("__TEXT,text_env"))) -# else -# define ESECT __attribute__ ((section("text_env"))) -# endif -#else -#define ESECT -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup compat Compatibility Macros - * A bunch of macros to minimize the amount of platform-specific ifdefs - * needed throughout the rest of the code. When the features this library - * needs are similar enough to POSIX to be hidden in a one-or-two line - * replacement, this macro approach is used. - * @{ - */ - - /** Features under development */ -#ifndef MDB_DEVEL -#define MDB_DEVEL 0 -#endif - - /** Wrapper around __func__, which is a C99 feature */ -#if __STDC_VERSION__ >= 199901L -# define mdb_func_ __func__ -#elif __GNUC__ >= 2 || _MSC_VER >= 1300 -# define mdb_func_ __FUNCTION__ -#else -/* If a debug message says (), update the #if statements above */ -# define mdb_func_ "" -#endif - -#ifdef _WIN32 -#define MDB_USE_HASH 1 -#define MDB_PIDLOCK 0 -#define THREAD_RET DWORD -#define pthread_t HANDLE -#define pthread_mutex_t HANDLE -#define pthread_cond_t HANDLE -#define pthread_key_t DWORD -#define pthread_self() GetCurrentThreadId() -#define pthread_key_create(x,y) \ - ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) -#define pthread_key_delete(x) TlsFree(x) -#define pthread_getspecific(x) TlsGetValue(x) -#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) -#define pthread_mutex_unlock(x) ReleaseMutex(*x) -#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) -#define pthread_cond_signal(x) SetEvent(*x) -#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) -#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,(LPTHREAD_START_ROUTINE)start,arg,0,NULL) -#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) -#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_rmutex) -#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_rmutex) -#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_wmutex) -#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_wmutex) -#define getpid() GetCurrentProcessId() -#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) -#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) -#define ErrCode() GetLastError() -#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} -#define close(fd) (CloseHandle(fd) ? 0 : -1) -#define munmap(ptr,len) UnmapViewOfFile(ptr) -#ifdef PROCESS_QUERY_LIMITED_INFORMATION -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION -#else -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 -#endif -#define Z "I" -#else -#define THREAD_RET void * -#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) -#define THREAD_FINISH(thr) pthread_join(thr,NULL) -#define Z "z" /**< printf format modifier for size_t */ - - /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ -#define MDB_PIDLOCK 1 - -#ifdef MDB_USE_POSIX_SEM - -#define LOCK_MUTEX_R(env) mdb_sem_wait((env)->me_rmutex) -#define UNLOCK_MUTEX_R(env) sem_post((env)->me_rmutex) -#define LOCK_MUTEX_W(env) mdb_sem_wait((env)->me_wmutex) -#define UNLOCK_MUTEX_W(env) sem_post((env)->me_wmutex) - -static int -mdb_sem_wait(sem_t *sem) -{ - int rc; - while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; - return rc; -} - -#else - /** Lock the reader mutex. - */ -#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_txns->mti_mutex) - /** Unlock the reader mutex. - */ -#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_txns->mti_mutex) - - /** Lock the writer mutex. - * Only a single write transaction is allowed at a time. Other writers - * will block waiting for this mutex. - */ -#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_txns->mti_wmutex) - /** Unlock the writer mutex. - */ -#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_txns->mti_wmutex) -#endif /* MDB_USE_POSIX_SEM */ - - /** Get the error code for the last failed system function. - */ -#define ErrCode() errno - - /** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#define HANDLE int - - /** A value for an invalid file handle. - * Mainly used to initialize file variables and signify that they are - * unused. - */ -#define INVALID_HANDLE_VALUE (-1) - - /** Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. - */ -#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) -#endif - -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) -#define MNAME_LEN 32 -#else -#define MNAME_LEN (sizeof(pthread_mutex_t)) -#endif - -/** @} */ - -#ifndef _WIN32 -/** A flag for opening a file and requesting synchronous data writes. - * This is only used when writing a meta page. It's not strictly needed; - * we could just do a normal write and then immediately perform a flush. - * But if this flag is available it saves us an extra system call. - * - * @note If O_DSYNC is undefined but exists in /usr/include, - * preferably set some compiler flag to get the definition. - * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. - */ -#ifndef MDB_DSYNC -# define MDB_DSYNC O_DSYNC -#endif -#endif - -/** Function for flushing the data of a file. Define this to fsync - * if fdatasync() is not supported. - */ -#ifndef MDB_FDATASYNC -# define MDB_FDATASYNC fdatasync -#endif - -#ifndef MDB_MSYNC -# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) -#endif - -#ifndef MS_SYNC -#define MS_SYNC 1 -#endif - -#ifndef MS_ASYNC -#define MS_ASYNC 0 -#endif - - /** A page number in the database. - * Note that 64 bit page numbers are overkill, since pages themselves - * already represent 12-13 bits of addressable memory, and the OS will - * always limit applications to a maximum of 63 bits of address space. - * - * @note In the #MDB_node structure, we only store 48 bits of this value, - * which thus limits us to only 60 bits of addressable data. - */ -typedef MDB_ID pgno_t; - - /** A transaction ID. - * See struct MDB_txn.mt_txnid for details. - */ -typedef MDB_ID txnid_t; - -/** @defgroup debug Debug Macros - * @{ - */ -#ifndef MDB_DEBUG - /** Enable debug output. Needs variable argument macros (a C99 feature). - * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs - * read from and written to the database (used for free space management). - */ -#define MDB_DEBUG 0 -#endif - -#if MDB_DEBUG -static int mdb_debug; -static txnid_t mdb_debug_start; - - /** Print a debug message with printf formatting. - * Requires double parenthesis around 2 or more args. - */ -# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) -# define DPRINTF0(fmt, ...) \ - fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) -#else -# define DPRINTF(args) ((void) 0) -#endif - /** Print a debug string. - * The string is printed literally, with no format processing. - */ -#define DPUTS(arg) DPRINTF(("%s", arg)) - /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) -/** @} */ - - /** @brief The maximum size of a database page. - * - * It is 32k or 64k, since value-PAGEBASE must fit in - * #MDB_page.%mp_upper. - * - * LMDB will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. - */ -#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) - - /** The minimum number of keys required in a database page. - * Setting this to a larger value will place a smaller bound on the - * maximum size of a data item. Data items larger than this size will - * be pushed into overflow pages instead of being stored directly in - * the B-tree node. This value used to default to 4. With a page size - * of 4096 bytes that meant that any item larger than 1024 bytes would - * go into an overflow page. That also meant that on average 2-3KB of - * each overflow page was wasted space. The value cannot be lower than - * 2 because then there would no longer be a tree structure. With this - * value, items larger than 2KB will go into overflow pages, and on - * average only 1KB will be wasted. - */ -#define MDB_MINKEYS 2 - - /** A stamp that identifies a file as an LMDB file. - * There's nothing special about this value other than that it is easily - * recognizable, and it will reflect any byte order mismatches. - */ -#define MDB_MAGIC 0xBEEFC0DE - - /** The version number for a database's datafile format. */ -#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) - /** The version number for a database's lockfile format. */ -#define MDB_LOCK_VERSION 1 - - /** @brief The max size of a key we can write, or 0 for dynamic max. - * - * Define this as 0 to compute the max from the page size. 511 - * is default for backwards compat: liblmdb <= 0.9.10 can break - * when modifying a DB with keys/dupsort data bigger than its max. - * #MDB_DEVEL sets the default to 0. - * - * Data items in an #MDB_DUPSORT database are also limited to - * this size, since they're actually keys of a sub-DB. Keys and - * #MDB_DUPSORT data items must fit on a node in a regular page. - */ -#ifndef MDB_MAXKEYSIZE -#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) -#endif - - /** The maximum size of a key we can write to the environment. */ -#if MDB_MAXKEYSIZE -#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) -#else -#define ENV_MAXKEY(env) ((env)->me_maxkey) -#endif - - /** @brief The maximum size of a data item. - * - * We only store a 32 bit value for node sizes. - */ -#define MAXDATASIZE 0xffffffffUL - -#if MDB_DEBUG - /** Key size which fits in a #DKBUF. - * @ingroup debug - */ -#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) - /** A key buffer. - * @ingroup debug - * This is used for printing a hex dump of a key's contents. - */ -#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] - /** Display a key in hex. - * @ingroup debug - * Invoke a function to display a key in hex. - */ -#define DKEY(x) mdb_dkey(x, kbuf) -#else -#define DKBUF -#define DKEY(x) 0 -#endif - - /** An invalid page number. - * Mainly used to denote an empty tree. - */ -#define P_INVALID (~(pgno_t)0) - - /** Test if the flags \b f are set in a flag word \b w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) - - /** Round \b n up to an even number. */ -#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ - - /** Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. - */ -typedef uint16_t indx_t; - - /** Default size of memory map. - * This is certainly too small for any actual applications. Apps should always set - * the size explicitly using #mdb_env_set_mapsize(). - */ -#define DEFAULT_MAPSIZE 1048576 - -/** @defgroup readers Reader Lock Table - * Readers don't acquire any locks for their data access. Instead, they - * simply record their transaction ID in the reader table. The reader - * mutex is needed just to find an empty slot in the reader table. The - * slot's address is saved in thread-specific data so that subsequent read - * transactions started by the same thread need no further locking to proceed. - * - * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. - * - * No reader table is used if the database is on a read-only filesystem, or - * if #MDB_NOLOCK is set. - * - * Since the database uses multi-version concurrency control, readers don't - * actually need any locking. This table is used to keep track of which - * readers are using data from which old transactions, so that we'll know - * when a particular old transaction is no longer in use. Old transactions - * that have discarded any data pages can then have those pages reclaimed - * for use by a later write transaction. - * - * The lock table is constructed such that reader slots are aligned with the - * processor's cache line size. Any slot is only ever used by one thread. - * This alignment guarantees that there will be no contention or cache - * thrashing as threads update their own slot info, and also eliminates - * any need for locking when accessing a slot. - * - * A writer thread will scan every slot in the table to determine the oldest - * outstanding reader transaction. Any freed pages older than this will be - * reclaimed by the writer. The writer doesn't use any locks when scanning - * this table. This means that there's no guarantee that the writer will - * see the most up-to-date reader info, but that's not required for correct - * operation - all we need is to know the upper bound on the oldest reader, - * we don't care at all about the newest reader. So the only consequence of - * reading stale information here is that old pages might hang around a - * while longer before being reclaimed. That's actually good anyway, because - * the longer we delay reclaiming old pages, the more likely it is that a - * string of contiguous pages can be found after coalescing old pages from - * many old transactions together. - * @{ - */ - /** Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. 126 readers plus a - * couple mutexes fit exactly into 8KB on my development machine. - * Applications should set the table size using #mdb_env_set_maxreaders(). - */ -#define DEFAULT_READERS 126 - - /** The size of a CPU cache line in bytes. We want our lock structures - * aligned to this size to avoid false cache line sharing in the - * lock table. - * This value works for most CPUs. For Itanium this should be 128. - */ -#ifndef CACHELINE -#define CACHELINE 64 -#endif - - /** The information we store in a single slot of the reader table. - * In addition to a transaction ID, we also record the process and - * thread ID that owns a slot, so that we can detect stale information, - * e.g. threads or processes that went away without cleaning up. - * @note We currently don't check for stale records. We simply re-init - * the table when we know that we're the only process opening the - * lock file. - */ -typedef struct MDB_rxbody { - /** Current Transaction ID when this transaction began, or (txnid_t)-1. - * Multiple readers that start at the same time will probably have the - * same ID here. Again, it's not important to exclude them from - * anything; all we need to know is which version of the DB they - * started from so we can avoid overwriting any data used in that - * particular version. - */ - volatile txnid_t mrb_txnid; - /** The process ID of the process owning this reader txn. */ - volatile MDB_PID_T mrb_pid; - /** The thread ID of the thread owning this txn. */ - volatile MDB_THR_T mrb_tid; -} MDB_rxbody; - - /** The actual reader record, with cacheline padding. */ -typedef struct MDB_reader { - union { - MDB_rxbody mrx; - /** shorthand for mrb_txnid */ -#define mr_txnid mru.mrx.mrb_txnid -#define mr_pid mru.mrx.mrb_pid -#define mr_tid mru.mrx.mrb_tid - /** cache line alignment */ - char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; - } mru; -} MDB_reader; - - /** The header for the reader table. - * The table resides in a memory-mapped file. (This is a different file - * than is used for the main database.) - * - * For POSIX the actual mutexes reside in the shared memory of this - * mapped file. On Windows, mutexes are named objects allocated by the - * kernel; we store the mutex names in this mapped file so that other - * processes can grab them. This same approach is also used on - * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support - * process-shared POSIX mutexes. For these cases where a named object - * is used, the object name is derived from a 64 bit FNV hash of the - * environment pathname. As such, naming collisions are extremely - * unlikely. If a collision occurs, the results are unpredictable. - */ -typedef struct MDB_txbody { - /** Stamp identifying this as an LMDB file. It must be set - * to #MDB_MAGIC. */ - uint32_t mtb_magic; - /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ - uint32_t mtb_format; -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - char mtb_rmname[MNAME_LEN]; -#else - /** Mutex protecting access to this table. - * This is the reader lock that #LOCK_MUTEX_R acquires. - */ - pthread_mutex_t mtb_mutex; -#endif - /** The ID of the last transaction committed to the database. - * This is recorded here only for convenience; the value can always - * be determined by reading the main database meta pages. - */ - volatile txnid_t mtb_txnid; - /** The number of slots that have been used in the reader table. - * This always records the maximum count, it is not decremented - * when readers release their slots. - */ - volatile unsigned mtb_numreaders; -} MDB_txbody; - - /** The actual reader table definition. */ -typedef struct MDB_txninfo { - union { - MDB_txbody mtb; -#define mti_magic mt1.mtb.mtb_magic -#define mti_format mt1.mtb.mtb_format -#define mti_mutex mt1.mtb.mtb_mutex -#define mti_rmname mt1.mtb.mtb_rmname -#define mti_txnid mt1.mtb.mtb_txnid -#define mti_numreaders mt1.mtb.mtb_numreaders - char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; - } mt1; - union { -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - char mt2_wmname[MNAME_LEN]; -#define mti_wmname mt2.mt2_wmname -#else - pthread_mutex_t mt2_wmutex; -#define mti_wmutex mt2.mt2_wmutex -#endif - char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; - } mt2; - MDB_reader mti_readers[1]; -} MDB_txninfo; - - /** Lockfile format signature: version, features and field layout */ -#define MDB_LOCK_FORMAT \ - ((uint32_t) \ - ((MDB_LOCK_VERSION) \ - /* Flags which describe functionality */ \ - + (((MDB_PIDLOCK) != 0) << 16))) -/** @} */ - -/** Common header for all page types. - * Overflow records occupy a number of contiguous pages with no - * headers on any page after the first. - */ -typedef struct MDB_page { -#define mp_pgno mp_p.p_pgno -#define mp_next mp_p.p_next - union { - pgno_t p_pgno; /**< page number */ - struct MDB_page *p_next; /**< for in-memory list of freed pages */ - } mp_p; - uint16_t mp_pad; -/** @defgroup mdb_page Page Flags - * @ingroup internal - * Flags for the page headers. - * @{ - */ -#define P_BRANCH 0x01 /**< branch page */ -#define P_LEAF 0x02 /**< leaf page */ -#define P_OVERFLOW 0x04 /**< overflow page */ -#define P_META 0x08 /**< meta page */ -#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ -#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ -#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ -#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ -#define P_KEEP 0x8000 /**< leave this page alone during spill */ -/** @} */ - uint16_t mp_flags; /**< @ref mdb_page */ -#define mp_lower mp_pb.pb.pb_lower -#define mp_upper mp_pb.pb.pb_upper -#define mp_pages mp_pb.pb_pages - union { - struct { - indx_t pb_lower; /**< lower bound of free space */ - indx_t pb_upper; /**< upper bound of free space */ - } pb; - uint32_t pb_pages; /**< number of overflow pages */ - } mp_pb; - indx_t mp_ptrs[1]; /**< dynamic size */ -} MDB_page; - - /** Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) - - /** Address of first usable data byte in a page, after the header */ -#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) - - /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ -#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) - - /** Number of nodes on a page */ -#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) - - /** The amount of space remaining in the page */ -#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) - - /** The percentage of space used in the page, in tenths of a percent. */ -#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ - ((env)->me_psize - PAGEHDRSZ)) - /** The minimum page fill factor, in tenths of a percent. - * Pages emptier than this are candidates for merging. - */ -#define FILL_THRESHOLD 250 - - /** Test if a page is a leaf page */ -#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) - /** Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) - /** Test if a page is a branch page */ -#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) - /** Test if a page is an overflow page */ -#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) - /** Test if a page is a sub page */ -#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) - - /** The number of overflow pages needed to store the given size. */ -#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - - /** Link in #MDB_txn.%mt_loose_pgs list */ -#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) - - /** Header for a single key/data pair within a page. - * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. - * We guarantee 2-byte alignment for 'MDB_node's. - */ -typedef struct MDB_node { - /** lo and hi are used for data size on leaf nodes and for - * child pgno on branch nodes. On 64 bit platforms, flags - * is also used for pgno. (Branch nodes have no flags). - * They are in host byte order in case that lets some - * accesses be optimized into a 32-bit word access. - */ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ -#else - unsigned short mn_hi, mn_lo; -#endif -/** @defgroup mdb_node Node Flags - * @ingroup internal - * Flags for node headers. - * @{ - */ -#define F_BIGDATA 0x01 /**< data put on overflow page */ -#define F_SUBDATA 0x02 /**< data is a sub-database */ -#define F_DUPDATA 0x04 /**< data has duplicates */ - -/** valid flags for #mdb_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) - -/** @} */ - unsigned short mn_flags; /**< @ref mdb_node */ - unsigned short mn_ksize; /**< key size */ - char mn_data[1]; /**< key and data are appended here */ -} MDB_node; - - /** Size of the node header, excluding dynamic data at the end */ -#define NODESIZE offsetof(MDB_node, mn_data) - - /** Bit position of top word in page number, for shifting mn_flags */ -#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) - - /** Size of a node in a branch page with a given key. - * This is just the node header plus the key, there is no data. - */ -#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) - - /** Size of a node in a leaf page with a given key and data. - * This is node header plus key plus data size. - */ -#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) - - /** Address of node \b i in page \b p */ -#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) - - /** Address of the key for the node */ -#define NODEKEY(node) (void *)((node)->mn_data) - - /** Address of the data for a node */ -#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) - - /** Get the page number pointed to by a branch node */ -#define NODEPGNO(node) \ - ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ - (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) - /** Set the page number in a branch node */ -#define SETPGNO(node,pgno) do { \ - (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ - if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) - - /** Get the size of the data in a leaf node */ -#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) - /** Set the size of the data for a leaf node */ -#define SETDSZ(node,size) do { \ - (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) - /** The size of a key in a node */ -#define NODEKSZ(node) ((node)->mn_ksize) - - /** Copy a page number from src to dst */ -#ifdef MISALIGNED_OK -#define COPY_PGNO(dst,src) dst = src -#else -#if SIZE_MAX > 4294967295UL -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d++ = *s++; \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#else -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#endif -#endif - /** The address of a key in a LEAF2 page. - * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. - * There are no node headers, keys are stored contiguously. - */ -#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) - - /** Set the \b node's key into \b keyptr, if requested. */ -#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ - (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } - - /** Set the \b node's key into \b key. */ -#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } - - /** Information about a single database in the environment. */ -typedef struct MDB_db { - uint32_t md_pad; /**< also ksize for LEAF2 pages */ - uint16_t md_flags; /**< @ref mdb_dbi_open */ - uint16_t md_depth; /**< depth of this tree */ - pgno_t md_branch_pages; /**< number of internal pages */ - pgno_t md_leaf_pages; /**< number of leaf pages */ - pgno_t md_overflow_pages; /**< number of overflow pages */ - size_t md_entries; /**< number of data items */ - pgno_t md_root; /**< the root page of this tree */ -} MDB_db; - - /** mdb_dbi_open flags */ -#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ -#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) -#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ - MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) - - /** Handle for the DB used to track free pages. */ -#define FREE_DBI 0 - /** Handle for the default DB. */ -#define MAIN_DBI 1 - - /** Meta page content. - * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). - */ -typedef struct MDB_meta { - /** Stamp identifying this as an LMDB file. It must be set - * to #MDB_MAGIC. */ - uint32_t mm_magic; - /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ - uint32_t mm_version; - void *mm_address; /**< address for fixed mapping */ - size_t mm_mapsize; /**< size of mmap region */ - MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ - /** The size of pages used in this DB */ -#define mm_psize mm_dbs[0].md_pad - /** Any persistent environment flags. @ref mdb_env */ -#define mm_flags mm_dbs[0].md_flags - pgno_t mm_last_pg; /**< last used page in file */ - volatile txnid_t mm_txnid; /**< txnid that committed this page */ -} MDB_meta; - - /** Buffer for a stack-allocated meta page. - * The members define size and alignment, and silence type - * aliasing warnings. They are not used directly; that could - * mean incorrectly using several union members in parallel. - */ -typedef union MDB_metabuf { - MDB_page mb_page; - struct { - char mm_pad[PAGEHDRSZ]; - MDB_meta mm_meta; - } mb_metabuf; -} MDB_metabuf; - - /** Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. - */ -typedef struct MDB_dbx { - MDB_val md_name; /**< name of the database */ - MDB_cmp_func *md_cmp; /**< function for comparing keys */ - MDB_cmp_func *md_dcmp; /**< function for comparing data items */ - MDB_rel_func *md_rel; /**< user relocate function */ - void *md_relctx; /**< user-provided context for md_rel */ -} MDB_dbx; - - /** A database transaction. - * Every operation requires a transaction handle. - */ -struct MDB_txn { - MDB_txn *mt_parent; /**< parent of a nested txn */ - MDB_txn *mt_child; /**< nested txn under this txn */ - pgno_t mt_next_pgno; /**< next unallocated page */ - /** The ID of this transaction. IDs are integers incrementing from 1. - * Only committed write transactions increment the ID. If a transaction - * aborts, the ID may be re-used by the next writer. - */ - txnid_t mt_txnid; - MDB_env *mt_env; /**< the DB environment */ - /** The list of pages that became unused during this transaction. - */ - MDB_IDL mt_free_pgs; - /** The list of loose pages that became unused and may be reused - * in this transaction, linked through #NEXT_LOOSE_PAGE(page). - */ - MDB_page *mt_loose_pgs; - /* #Number of loose pages (#mt_loose_pgs) */ - int mt_loose_count; - /** The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. - */ - MDB_IDL mt_spill_pgs; - union { - /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ - MDB_ID2L dirty_list; - /** For read txns: This thread/txn's reader table slot, or NULL. */ - MDB_reader *reader; - } mt_u; - /** Array of records for each DB known in the environment. */ - MDB_dbx *mt_dbxs; - /** Array of MDB_db records for each known DB */ - MDB_db *mt_dbs; - /** Array of sequence numbers for each DB handle */ - unsigned int *mt_dbiseqs; -/** @defgroup mt_dbflag Transaction DB Flags - * @ingroup internal - * @{ - */ -#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ -#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ -#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ -#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ -/** @} */ - /** In write txns, array of cursors for each DB */ - MDB_cursor **mt_cursors; - /** Array of flags for each DB */ - unsigned char *mt_dbflags; - /** Number of DB records in use. This number only ever increments; - * we don't decrement it when individual DB handles are closed. - */ - MDB_dbi mt_numdbs; - -/** @defgroup mdb_txn Transaction Flags - * @ingroup internal - * @{ - */ -#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ -#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ -#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ -#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ -/** @} */ - unsigned int mt_flags; /**< @ref mdb_txn */ - /** #dirty_list room: Array size - \#dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirty_list into mt_parent after freeing hidden mt_parent pages. - */ - unsigned int mt_dirty_room; -}; - -/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. - * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to - * raise this on a 64 bit machine. - */ -#define CURSOR_STACK 32 - -struct MDB_xcursor; - - /** Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. #MDB_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a #P_SUBP page can be stale. - * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). - */ -struct MDB_cursor { - /** Next cursor on this DB in this txn */ - MDB_cursor *mc_next; - /** Backup of the original cursor if this cursor is a shadow */ - MDB_cursor *mc_backup; - /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ - struct MDB_xcursor *mc_xcursor; - /** The transaction that owns this cursor */ - MDB_txn *mc_txn; - /** The database handle this cursor operates on */ - MDB_dbi mc_dbi; - /** The database record for this cursor */ - MDB_db *mc_db; - /** The database auxiliary record for this cursor */ - MDB_dbx *mc_dbx; - /** The @ref mt_dbflag for this database */ - unsigned char *mc_dbflag; - unsigned short mc_snum; /**< number of pushed pages */ - unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ -/** @defgroup mdb_cursor Cursor Flags - * @ingroup internal - * Cursor state flags. - * @{ - */ -#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ -#define C_EOF 0x02 /**< No more data */ -#define C_SUB 0x04 /**< Cursor is a sub-cursor */ -#define C_DEL 0x08 /**< last op was a cursor_del */ -#define C_SPLITTING 0x20 /**< Cursor is in page_split */ -#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ -/** @} */ - unsigned int mc_flags; /**< @ref mdb_cursor */ - MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ -}; - - /** Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. - */ -typedef struct MDB_xcursor { - /** A sub-cursor for traversing the Dup DB */ - MDB_cursor mx_cursor; - /** The database record for this Dup DB */ - MDB_db mx_db; - /** The auxiliary DB record for this Dup DB */ - MDB_dbx mx_dbx; - /** The @ref mt_dbflag for this Dup DB */ - unsigned char mx_dbflag; -} MDB_xcursor; - - /** State of FreeDB old pages, stored in the MDB_env */ -typedef struct MDB_pgstate { - pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ - txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ -} MDB_pgstate; - - /** The database environment. */ -struct MDB_env { - HANDLE me_fd; /**< The main data file */ - HANDLE me_lfd; /**< The lock file */ - HANDLE me_mfd; /**< just for writing the meta pages */ - /** Failed to update the meta page. Probably an I/O error. */ -#define MDB_FATAL_ERROR 0x80000000U - /** Some fields are initialized. */ -#define MDB_ENV_ACTIVE 0x20000000U - /** me_txkey is set */ -#define MDB_ENV_TXKEY 0x10000000U - /** fdatasync is unreliable */ -#define MDB_FSYNCONLY 0x08000000U - uint32_t me_flags; /**< @ref mdb_env */ - unsigned int me_psize; /**< DB page size, inited from me_os_psize */ - unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ - unsigned int me_maxreaders; /**< size of the reader table */ - unsigned int me_numreaders; /**< max numreaders set by this env */ - MDB_dbi me_numdbs; /**< number of DBs opened */ - MDB_dbi me_maxdbs; /**< size of the DB table */ - MDB_PID_T me_pid; /**< process ID of this env */ - char *me_path; /**< path to the DB files */ - char *me_map; /**< the memory map of the data file */ - MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ - MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ - void *me_pbuf; /**< scratch area for DUPSORT put() */ - MDB_txn *me_txn; /**< current write transaction */ - MDB_txn *me_txn0; /**< prealloc'd write transaction */ - size_t me_mapsize; /**< size of the data memory map */ - off_t me_size; /**< current file size */ - pgno_t me_maxpg; /**< me_mapsize / me_psize */ - MDB_dbx *me_dbxs; /**< array of static DB info */ - uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ - unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ - pthread_key_t me_txkey; /**< thread-key for readers */ - txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ - MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ -# define me_pglast me_pgstate.mf_pglast -# define me_pghead me_pgstate.mf_pghead - MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ - /** IDL of pages that became unused in a write txn */ - MDB_IDL me_free_pgs; - /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ - MDB_ID2L me_dirty_list; - /** Max number of freelist items that can fit in a single overflow page */ - int me_maxfree_1pg; - /** Max size of a node on a page */ - unsigned int me_nodemax; -#if !(MDB_MAXKEYSIZE) - unsigned int me_maxkey; /**< max size of a key */ -#endif - int me_live_reader; /**< have liveness lock in reader table */ -#ifdef _WIN32 - int me_pidquery; /**< Used in OpenProcess */ - HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ - HANDLE me_wmutex; -#elif defined(MDB_USE_POSIX_SEM) - sem_t *me_rmutex; /* Shared mutexes are not supported */ - sem_t *me_wmutex; -#endif - void *me_userctx; /**< User-settable context */ - MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ -}; - - /** Nested transaction */ -typedef struct MDB_ntxn { - MDB_txn mnt_txn; /**< the transaction */ - MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ -} MDB_ntxn; - - /** max number of pages to commit in one writev() call */ -#define MDB_COMMIT_PAGES 64 -#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES -#undef MDB_COMMIT_PAGES -#define MDB_COMMIT_PAGES IOV_MAX -#endif - - /** max bytes to write in one call */ -#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) - - /** Check \b txn and \b dbi arguments to a function */ -#define TXN_DBI_EXIST(txn, dbi) \ - ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) - - /** Check for misused \b dbi handles */ -#define TXN_DBI_CHANGED(txn, dbi) \ - ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) - -static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); -static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); -static int mdb_page_touch(MDB_cursor *mc); - -static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); -static int mdb_page_search_root(MDB_cursor *mc, - MDB_val *key, int modify); -#define MDB_PS_MODIFY 1 -#define MDB_PS_ROOTONLY 2 -#define MDB_PS_FIRST 4 -#define MDB_PS_LAST 8 -static int mdb_page_search(MDB_cursor *mc, - MDB_val *key, int flags); -static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); - -#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ -static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, - pgno_t newpgno, unsigned int nflags); - -static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); -static int mdb_env_pick_meta(const MDB_env *env); -static int mdb_env_write_meta(MDB_txn *txn); -#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ -# define mdb_env_close0(env, excl) mdb_env_close1(env) -#endif -static void mdb_env_close0(MDB_env *env, int excl); - -static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); -static int mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); -static void mdb_node_del(MDB_cursor *mc, int ksize); -static void mdb_node_shrink(MDB_page *mp, indx_t indx); -static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); -static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); -static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); -static size_t mdb_branch_size(MDB_env *env, MDB_val *key); - -static int mdb_rebalance(MDB_cursor *mc); -static int mdb_update_key(MDB_cursor *mc, MDB_val *key); - -static void mdb_cursor_pop(MDB_cursor *mc); -static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); - -static int mdb_cursor_del0(MDB_cursor *mc); -static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); -static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); -static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, - int *exactp); -static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); -static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); - -static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); -static void mdb_xcursor_init0(MDB_cursor *mc); -static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); - -static int mdb_drop0(MDB_cursor *mc, int subs); -static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); - -/** @cond */ -static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; -/** @endcond */ - -#ifdef _WIN32 -static SECURITY_DESCRIPTOR mdb_null_sd; -static SECURITY_ATTRIBUTES mdb_all_sa; -static int mdb_sec_inited; -#endif - -/** Return the library version info. */ -char * -mdb_version(int *major, int *minor, int *patch) -{ - if (major) *major = MDB_VERSION_MAJOR; - if (minor) *minor = MDB_VERSION_MINOR; - if (patch) *patch = MDB_VERSION_PATCH; - return MDB_VERSION_STRING; -} - -/** Table of descriptions for LMDB @ref errors */ -static char *const mdb_errstr[] = { - "MDB_KEYEXIST: Key/data pair already exists", - "MDB_NOTFOUND: No matching key/data pair found", - "MDB_PAGE_NOTFOUND: Requested page not found", - "MDB_CORRUPTED: Located page was wrong type", - "MDB_PANIC: Update of meta page failed", - "MDB_VERSION_MISMATCH: Database environment version mismatch", - "MDB_INVALID: File is not an LMDB file", - "MDB_MAP_FULL: Environment mapsize limit reached", - "MDB_DBS_FULL: Environment maxdbs limit reached", - "MDB_READERS_FULL: Environment maxreaders limit reached", - "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", - "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", - "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", - "MDB_PAGE_FULL: Internal error - page has no more space", - "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", - "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", - "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", - "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", - "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", - "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", -}; - -char * -mdb_strerror(int err) -{ -#ifdef _WIN32 - /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. - * This works as long as no function between the call to mdb_strerror - * and the actual use of the message uses more than 4K of stack. - */ - char pad[4096]; - char buf[1024], *ptr = buf; -#endif - int i; - if (!err) - return ("Successful return: 0"); - - if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { - i = err - MDB_KEYEXIST; - return mdb_errstr[i]; - } - -#ifdef _WIN32 - /* These are the C-runtime error codes we use. The comment indicates - * their numeric value, and the Win32 error they would correspond to - * if the error actually came from a Win32 API. A major mess, we should - * have used LMDB-specific error codes for everything. - */ - switch(err) { - case ENOENT: /* 2, FILE_NOT_FOUND */ - case EIO: /* 5, ACCESS_DENIED */ - case ENOMEM: /* 12, INVALID_ACCESS */ - case EACCES: /* 13, INVALID_DATA */ - case EBUSY: /* 16, CURRENT_DIRECTORY */ - case EINVAL: /* 22, BAD_COMMAND */ - case ENOSPC: /* 28, OUT_OF_PAPER */ - return strerror(err); - default: - ; - } - buf[0] = 0; - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, 0, ptr, sizeof(buf), (va_list *)pad); - return ptr; -#else - return strerror(err); -#endif -} - -/** assert(3) variant in cursor context */ -#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) -/** assert(3) variant in transaction context */ -#define mdb_tassert(mc, expr) mdb_assert0((txn)->mt_env, expr, #expr) -/** assert(3) variant in environment context */ -#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) - -#ifndef NDEBUG -# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ - mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) - -static void -mdb_assert_fail(MDB_env *env, const char *expr_txt, - const char *func, const char *file, int line) -{ - char buf[400]; - sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", - file, line, expr_txt, func); - if (env->me_assert_func) - env->me_assert_func(env, buf); - fprintf(stderr, "%s\n", buf); - abort(); -} -#else -# define mdb_assert0(env, expr, expr_txt) ((void) 0) -#endif /* NDEBUG */ - -#if MDB_DEBUG -/** Return the page number of \b mp which may be sub-page, for debug output */ -static pgno_t -mdb_dbg_pgno(MDB_page *mp) -{ - pgno_t ret; - COPY_PGNO(ret, mp->mp_pgno); - return ret; -} - -/** Display a key in hexadecimal and return the address of the result. - * @param[in] key the key to display - * @param[in] buf the buffer to write into. Should always be #DKBUF. - * @return The key in hexadecimal form. - */ -char * -mdb_dkey(MDB_val *key, char *buf) -{ - char *ptr = buf; - unsigned char *c = key->mv_data; - unsigned int i; - - if (!key) - return ""; - - if (key->mv_size > DKBUF_MAXKEYSIZE) - return "MDB_MAXKEYSIZE"; - /* may want to make this a dynamic check: if the key is mostly - * printable characters, print it as-is instead of converting to hex. - */ -#if 1 - buf[0] = '\0'; - for (i=0; imv_size; i++) - ptr += sprintf(ptr, "%02x", *c++); -#else - sprintf(buf, "%.*s", key->mv_size, key->mv_data); -#endif - return buf; -} - -static const char * -mdb_leafnode_type(MDB_node *n) -{ - static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; - return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : - tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; -} - -/** Display all the keys in the page. */ -void -mdb_page_list(MDB_page *mp) -{ - pgno_t pgno = mdb_dbg_pgno(mp); - const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; - MDB_node *node; - unsigned int i, nkeys, nsize, total = 0; - MDB_val key; - DKBUF; - - switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { - case P_BRANCH: type = "Branch page"; break; - case P_LEAF: type = "Leaf page"; break; - case P_LEAF|P_SUBP: type = "Sub-page"; break; - case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; - case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; - case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", - pgno, mp->mp_pages, state); - return; - case P_META: - fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", - pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); - return; - default: - fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags); - return; - } - - nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); - - for (i=0; imp_pad; - key.mv_data = LEAF2KEY(mp, i, nsize); - total += nsize; - fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); - continue; - } - node = NODEPTR(mp, i); - key.mv_size = node->mn_ksize; - key.mv_data = node->mn_data; - nsize = NODESIZE + key.mv_size; - if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), - DKEY(&key)); - total += nsize; - } else { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - nsize += sizeof(pgno_t); - else - nsize += NODEDSZ(node); - total += nsize; - nsize += sizeof(indx_t); - fprintf(stderr, "key %d: nsize %d, %s%s\n", - i, nsize, DKEY(&key), mdb_leafnode_type(node)); - } - total = EVEN(total); - } - fprintf(stderr, "Total: header %d + contents %d + unused %d\n", - IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); -} - -void -mdb_cursor_chk(MDB_cursor *mc) -{ - unsigned int i; - MDB_node *node; - MDB_page *mp; - - if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; - for (i=0; imc_top; i++) { - mp = mc->mc_pg[i]; - node = NODEPTR(mp, mc->mc_ki[i]); - if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) - printf("oops!\n"); - } - if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) - printf("ack!\n"); -} -#endif - -#if (MDB_DEBUG) > 2 -/** Count all the pages in each DB and in the freelist - * and make sure it matches the actual number of pages - * being used. - * All named DBs must be open for a correct count. - */ -static void mdb_audit(MDB_txn *txn) -{ - MDB_cursor mc; - MDB_val key, data; - MDB_ID freecount, count; - MDB_dbi i; - int rc; - - freecount = 0; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - mdb_tassert(txn, rc == MDB_NOTFOUND); - - count = 0; - for (i = 0; imt_numdbs; i++) { - MDB_xcursor mx; - if (!(txn->mt_dbflags[i] & DB_VALID)) - continue; - mdb_cursor_init(&mc, txn, i, &mx); - if (txn->mt_dbs[i].md_root == P_INVALID) - continue; - count += txn->mt_dbs[i].md_branch_pages + - txn->mt_dbs[i].md_leaf_pages + - txn->mt_dbs[i].md_overflow_pages; - if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); - for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { - unsigned j; - MDB_page *mp; - mp = mc.mc_pg[mc.mc_top]; - for (j=0; jmn_flags & F_SUBDATA) { - MDB_db db; - memcpy(&db, NODEDATA(leaf), sizeof(db)); - count += db.md_branch_pages + db.md_leaf_pages + - db.md_overflow_pages; - } - } - } - mdb_tassert(txn, rc == MDB_NOTFOUND); - } - } - if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { - fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", - txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); - } -} -#endif - -int -mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - return txn->mt_dbxs[dbi].md_cmp(a, b); -} - -int -mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - return txn->mt_dbxs[dbi].md_dcmp(a, b); -} - -/** Allocate memory for a page. - * Re-use old malloc'd pages first for singletons, otherwise just malloc. - */ -static MDB_page * -mdb_page_malloc(MDB_txn *txn, unsigned num) -{ - MDB_env *env = txn->mt_env; - MDB_page *ret = env->me_dpages; - size_t psize = env->me_psize, sz = psize, off; - /* For ! #MDB_NOMEMINIT, psize counts how much to init. - * For a single page alloc, we init everything after the page header. - * For multi-page, we init the final page; if the caller needed that - * many pages they will be filling in at least up to the last page. - */ - if (num == 1) { - if (ret) { - VGMEMP_ALLOC(env, ret, sz); - VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); - env->me_dpages = ret->mp_next; - return ret; - } - psize -= off = PAGEHDRSZ; - } else { - sz *= num; - off = sz - psize; - } - if ((ret = malloc(sz)) != NULL) { - VGMEMP_ALLOC(env, ret, sz); - if (!(env->me_flags & MDB_NOMEMINIT)) { - memset((char *)ret + off, 0, psize); - ret->mp_pad = 0; - } - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - return ret; -} -/** Free a single page. - * Saves single pages to a list, for future reuse. - * (This is not used for multi-page overflow pages.) - */ -static void -mdb_page_free(MDB_env *env, MDB_page *mp) -{ - mp->mp_next = env->me_dpages; - VGMEMP_FREE(env, mp); - env->me_dpages = mp; -} - -/** Free a dirty page */ -static void -mdb_dpage_free(MDB_env *env, MDB_page *dp) -{ - if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { - mdb_page_free(env, dp); - } else { - /* large pages just get freed directly */ - VGMEMP_FREE(env, dp); - free(dp); - } -} - -/** Return all dirty pages to dpage list */ -static void -mdb_dlist_free(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned i, n = dl[0].mid; - - for (i = 1; i <= n; i++) { - mdb_dpage_free(env, dl[i].mptr); - } - dl[0].mid = 0; -} - -/** Loosen or free a single page. - * Saves single pages to a list for future reuse - * in this same txn. It has been pulled from the freeDB - * and already resides on the dirty list, but has been - * deleted. Use these pages first before pulling again - * from the freeDB. - * - * If the page wasn't dirtied in this txn, just add it - * to this txn's free list. - */ -static int -mdb_page_loose(MDB_cursor *mc, MDB_page *mp) -{ - int loose = 0; - pgno_t pgno = mp->mp_pgno; - MDB_txn *txn = mc->mc_txn; - - if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { - if (txn->mt_parent) { - MDB_ID2 *dl = txn->mt_u.dirty_list; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - /* ok, it's ours */ - loose = 1; - } - } - } else { - /* no parent txn, so it's just ours */ - loose = 1; - } - } - if (loose) { - DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), - mp->mp_pgno)); - NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; - txn->mt_loose_pgs = mp; - txn->mt_loose_count++; - mp->mp_flags |= P_LOOSE; - } else { - int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); - if (rc) - return rc; - } - - return MDB_SUCCESS; -} - -/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. - * @param[in] mc A cursor handle for the current operation. - * @param[in] pflags Flags of the pages to update: - * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. - * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). - * @return 0 on success, non-zero on failure. - */ -static int -mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) -{ - enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m3; - MDB_xcursor *mx; - MDB_page *dp, *mp; - MDB_node *leaf; - unsigned i, j; - int rc = MDB_SUCCESS, level; - - /* Mark pages seen by cursors */ - if (mc->mc_flags & C_UNTRACK) - mc = NULL; /* will find mc in mt_cursors */ - for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { - for (; mc; mc=mc->mc_next) { - if (!(mc->mc_flags & C_INITIALIZED)) - continue; - for (m3 = mc;; m3 = &mx->mx_cursor) { - mp = NULL; - for (j=0; jmc_snum; j++) { - mp = m3->mc_pg[j]; - if ((mp->mp_flags & Mask) == pflags) - mp->mp_flags ^= P_KEEP; - } - mx = m3->mc_xcursor; - /* Proceed to mx if it is at a sub-database */ - if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) - break; - if (! (mp && (mp->mp_flags & P_LEAF))) - break; - leaf = NODEPTR(mp, m3->mc_ki[j-1]); - if (!(leaf->mn_flags & F_SUBDATA)) - break; - } - } - if (i == 0) - break; - } - - if (all) { - /* Mark dirty root pages */ - for (i=0; imt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - pgno_t pgno = txn->mt_dbs[i].md_root; - if (pgno == P_INVALID) - continue; - if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS) - break; - if ((dp->mp_flags & Mask) == pflags && level <= 1) - dp->mp_flags ^= P_KEEP; - } - } - } - - return rc; -} - -static int mdb_page_flush(MDB_txn *txn, int keep); - -/** Spill pages from the dirty list back to disk. - * This is intended to prevent running into #MDB_TXN_FULL situations, - * but note that they may still occur in a few cases: - * 1) our estimate of the txn size could be too small. Currently this - * seems unlikely, except with a large number of #MDB_MULTIPLE items. - * 2) child txns may run out of space if their parents dirtied a - * lot of pages and never spilled them. TODO: we probably should do - * a preemptive spill during #mdb_txn_begin() of a child txn, if - * the parent's dirty_room is below a given threshold. - * - * Otherwise, if not using nested txns, it is expected that apps will - * not run into #MDB_TXN_FULL any more. The pages are flushed to disk - * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. - * If the txn never references them again, they can be left alone. - * If the txn only reads them, they can be used without any fuss. - * If the txn writes them again, they can be dirtied immediately without - * going thru all of the work of #mdb_page_touch(). Such references are - * handled by #mdb_page_unspill(). - * - * Also note, we never spill DB root pages, nor pages of active cursors, - * because we'll need these back again soon anyway. And in nested txns, - * we can't spill a page in a child txn if it was already spilled in a - * parent txn. That would alter the parent txns' data even though - * the child hasn't committed yet, and we'd have no way to undo it if - * the child aborted. - * - * @param[in] m0 cursor A cursor handle identifying the transaction and - * database for which we are checking space. - * @param[in] key For a put operation, the key being stored. - * @param[in] data For a put operation, the data being stored. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) -{ - MDB_txn *txn = m0->mc_txn; - MDB_page *dp; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned int i, j, need; - int rc; - - if (m0->mc_flags & C_SUB) - return MDB_SUCCESS; - - /* Estimate how much space this op will take */ - i = m0->mc_db->md_depth; - /* Named DBs also dirty the main DB */ - if (m0->mc_dbi > MAIN_DBI) - i += txn->mt_dbs[MAIN_DBI].md_depth; - /* For puts, roughly factor in the key+data size */ - if (key) - i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; - i += i; /* double it for good measure */ - need = i; - - if (txn->mt_dirty_room > i) - return MDB_SUCCESS; - - if (!txn->mt_spill_pgs) { - txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); - if (!txn->mt_spill_pgs) - return ENOMEM; - } else { - /* purge deleted slots */ - MDB_IDL sl = txn->mt_spill_pgs; - unsigned int num = sl[0]; - j=0; - for (i=1; i<=num; i++) { - if (!(sl[i] & 1)) - sl[++j] = sl[i]; - } - sl[0] = j; - } - - /* Preserve pages which may soon be dirtied again */ - if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) - goto done; - - /* Less aggressive spill - we originally spilled the entire dirty list, - * with a few exceptions for cursor pages and DB root pages. But this - * turns out to be a lot of wasted effort because in a large txn many - * of those pages will need to be used again. So now we spill only 1/8th - * of the dirty pages. Testing revealed this to be a good tradeoff, - * better than 1/2, 1/4, or 1/10. - */ - if (need < MDB_IDL_UM_MAX / 8) - need = MDB_IDL_UM_MAX / 8; - - /* Save the page IDs of all the pages we're flushing */ - /* flush from the tail forward, this saves a lot of shifting later on. */ - for (i=dl[0].mid; i && need; i--) { - MDB_ID pn = dl[i].mid << 1; - dp = dl[i].mptr; - if (dp->mp_flags & (P_LOOSE|P_KEEP)) - continue; - /* Can't spill twice, make sure it's not already in a parent's - * spill list. - */ - if (txn->mt_parent) { - MDB_txn *tx2; - for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { - if (tx2->mt_spill_pgs) { - j = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { - dp->mp_flags |= P_KEEP; - break; - } - } - } - if (tx2) - continue; - } - if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) - goto done; - need--; - } - mdb_midl_sort(txn->mt_spill_pgs); - - /* Flush the spilled part of dirty list */ - if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) - goto done; - - /* Reset any dirty pages we kept that page_flush didn't see */ - rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); - -done: - txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; - return rc; -} - -/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ -static txnid_t -mdb_find_oldest(MDB_txn *txn) -{ - int i; - txnid_t mr, oldest = txn->mt_txnid - 1; - if (txn->mt_env->me_txns) { - MDB_reader *r = txn->mt_env->me_txns->mti_readers; - for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { - if (r[i].mr_pid) { - mr = r[i].mr_txnid; - if (oldest > mr) - oldest = mr; - } - } - } - return oldest; -} - -/** Add a page to the txn's dirty list */ -static void -mdb_page_dirty(MDB_txn *txn, MDB_page *mp) -{ - MDB_ID2 mid; - int rc, (*insert)(MDB_ID2L, MDB_ID2 *); - - if (txn->mt_env->me_flags & MDB_WRITEMAP) { - insert = mdb_mid2l_append; - } else { - insert = mdb_mid2l_insert; - } - mid.mid = mp->mp_pgno; - mid.mptr = mp; - rc = insert(txn->mt_u.dirty_list, &mid); - mdb_tassert(txn, rc == 0); - txn->mt_dirty_room--; -} - -/** Allocate page numbers and memory for writing. Maintain me_pglast, - * me_pghead and mt_next_pgno. - * - * If there are free pages available from older transactions, they - * are re-used first. Otherwise allocate a new page at mt_next_pgno. - * Do not modify the freedB, just merge freeDB records into me_pghead[] - * and move me_pglast to say which records were consumed. Only this - * function can create me_pghead and move me_pglast/mt_next_pgno. - * @param[in] mc cursor A cursor handle identifying the transaction and - * database for which we are allocating. - * @param[in] num the number of pages to allocate. - * @param[out] mp Address of the allocated page(s). Requests for multiple pages - * will always be satisfied by a single contiguous chunk of memory. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) -{ -#ifdef MDB_PARANOID /* Seems like we can ignore this now */ - /* Get at most more freeDB records once me_pghead - * has enough pages. If not enough, use new pages from the map. - * If and mc is updating the freeDB, only get new - * records if me_pghead is empty. Then the freelist cannot play - * catch-up with itself by growing while trying to save it. - */ - enum { Paranoid = 1, Max_retries = 500 }; -#else - enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; -#endif - int rc, retry = num * 60; - MDB_txn *txn = mc->mc_txn; - MDB_env *env = txn->mt_env; - pgno_t pgno, *mop = env->me_pghead; - unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; - MDB_page *np; - txnid_t oldest = 0, last; - MDB_cursor_op op; - MDB_cursor m2; - int found_old = 0; - - /* If there are any loose pages, just use them */ - if (num == 1 && txn->mt_loose_pgs) { - np = txn->mt_loose_pgs; - txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); - txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), - np->mp_pgno)); - *mp = np; - return MDB_SUCCESS; - } - - *mp = NULL; - - /* If our dirty list is already full, we can't do anything */ - if (txn->mt_dirty_room == 0) { - rc = MDB_TXN_FULL; - goto fail; - } - - for (op = MDB_FIRST;; op = MDB_NEXT) { - MDB_val key, data; - MDB_node *leaf; - pgno_t *idl; - - /* Seek a big enough contiguous page range. Prefer - * pages at the tail, just truncating the list. - */ - if (mop_len > n2) { - i = mop_len; - do { - pgno = mop[i]; - if (mop[i-n2] == pgno+n2) - goto search_done; - } while (--i > n2); - if (--retry < 0) - break; - } - - if (op == MDB_FIRST) { /* 1st iteration */ - /* Prepare to fetch more and coalesce */ - last = env->me_pglast; - oldest = env->me_pgoldest; - mdb_cursor_init(&m2, txn, FREE_DBI, NULL); - if (last) { - op = MDB_SET_RANGE; - key.mv_data = &last; /* will look up last+1 */ - key.mv_size = sizeof(last); - } - if (Paranoid && mc->mc_dbi == FREE_DBI) - retry = -1; - } - if (Paranoid && retry < 0 && mop_len) - break; - - last++; - /* Do not fetch more if the record will be too recent */ - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - rc = mdb_cursor_get(&m2, &key, NULL, op); - if (rc) { - if (rc == MDB_NOTFOUND) - break; - goto fail; - } - last = *(txnid_t*)key.mv_data; - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - np = m2.mc_pg[m2.mc_top]; - leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); - if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) - return rc; - - idl = (MDB_ID *) data.mv_data; - i = idl[0]; - if (!mop) { - if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { - rc = ENOMEM; - goto fail; - } - } else { - if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) - goto fail; - mop = env->me_pghead; - } - env->me_pglast = last; -#if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", - last, txn->mt_dbs[FREE_DBI].md_root, i)); - for (j = i; j; j--) - DPRINTF(("IDL %"Z"u", idl[j])); -#endif - /* Merge in descending sorted order */ - mdb_midl_xmerge(mop, idl); - mop_len = mop[0]; - } - - /* Use new pages from the map when nothing suitable in the freeDB */ - i = 0; - pgno = txn->mt_next_pgno; - if (pgno + num >= env->me_maxpg) { - DPUTS("DB size maxed out"); - rc = MDB_MAP_FULL; - goto fail; - } - -search_done: - if (env->me_flags & MDB_WRITEMAP) { - np = (MDB_page *)(env->me_map + env->me_psize * pgno); - } else { - if (!(np = mdb_page_malloc(txn, num))) { - rc = ENOMEM; - goto fail; - } - } - if (i) { - mop[0] = mop_len -= num; - /* Move any stragglers down */ - for (j = i-num; j < mop_len; ) - mop[++j] = mop[++i]; - } else { - txn->mt_next_pgno = pgno + num; - } - np->mp_pgno = pgno; - mdb_page_dirty(txn, np); - *mp = np; - - return MDB_SUCCESS; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Copy the used portions of a non-overflow page. - * @param[in] dst page to copy into - * @param[in] src page to copy from - * @param[in] psize size of a page - */ -static void -mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) -{ - enum { Align = sizeof(pgno_t) }; - indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; - - /* If page isn't full, just copy the used portion. Adjust - * alignment so memcpy may copy words instead of bytes. - */ - if ((unused &= -Align) && !IS_LEAF2(src)) { - upper = (upper + PAGEBASE) & -Align; - memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); - memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), - psize - upper); - } else { - memcpy(dst, src, psize - unused); - } -} - -/** Pull a page off the txn's spill list, if present. - * If a page being referenced was spilled to disk in this txn, bring - * it back and make it dirty/writable again. - * @param[in] txn the transaction handle. - * @param[in] mp the page being referenced. It must not be dirty. - * @param[out] ret the writable page, if any. ret is unchanged if - * mp wasn't spilled. - */ -static int -mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) -{ - MDB_env *env = txn->mt_env; - const MDB_txn *tx2; - unsigned x; - pgno_t pgno = mp->mp_pgno, pn = pgno << 1; - - for (tx2 = txn; tx2; tx2=tx2->mt_parent) { - if (!tx2->mt_spill_pgs) - continue; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - MDB_page *np; - int num; - if (txn->mt_dirty_room == 0) - return MDB_TXN_FULL; - if (IS_OVERFLOW(mp)) - num = mp->mp_pages; - else - num = 1; - if (env->me_flags & MDB_WRITEMAP) { - np = mp; - } else { - np = mdb_page_malloc(txn, num); - if (!np) - return ENOMEM; - if (num > 1) - memcpy(np, mp, num * env->me_psize); - else - mdb_page_copy(np, mp, env->me_psize); - } - if (tx2 == txn) { - /* If in current txn, this page is no longer spilled. - * If it happens to be the last page, truncate the spill list. - * Otherwise mark it as deleted by setting the LSB. - */ - if (x == txn->mt_spill_pgs[0]) - txn->mt_spill_pgs[0]--; - else - txn->mt_spill_pgs[x] |= 1; - } /* otherwise, if belonging to a parent txn, the - * page remains spilled until child commits - */ - - mdb_page_dirty(txn, np); - np->mp_flags |= P_DIRTY; - *ret = np; - break; - } - } - return MDB_SUCCESS; -} - -/** Touch a page: make it dirty and re-insert into tree with updated pgno. - * @param[in] mc cursor pointing to the page to be touched - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_touch(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top], *np; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m2, *m3; - pgno_t pgno; - int rc; - - if (!F_ISSET(mp->mp_flags, P_DIRTY)) { - if (txn->mt_flags & MDB_TXN_SPILLS) { - np = NULL; - rc = mdb_page_unspill(txn, mp, &np); - if (rc) - goto fail; - if (np) - goto done; - } - if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || - (rc = mdb_page_alloc(mc, 1, &np))) - goto fail; - pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), - mp->mp_pgno, pgno)); - mdb_cassert(mc, mp->mp_pgno != pgno); - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - /* Update the parent page, if any, to point to the new page */ - if (mc->mc_top) { - MDB_page *parent = mc->mc_pg[mc->mc_top-1]; - MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); - SETPGNO(node, pgno); - } else { - mc->mc_db->md_root = pgno; - } - } else if (txn->mt_parent && !IS_SUBP(mp)) { - MDB_ID2 mid, *dl = txn->mt_u.dirty_list; - pgno = mp->mp_pgno; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - return 0; - } - } - mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); - /* No - copy it */ - np = mdb_page_malloc(txn, 1); - if (!np) - return ENOMEM; - mid.mid = pgno; - mid.mptr = np; - rc = mdb_mid2l_insert(dl, &mid); - mdb_cassert(mc, rc == 0); - } else { - return 0; - } - - mdb_page_copy(np, mp, txn->mt_env->me_psize); - np->mp_pgno = pgno; - np->mp_flags |= P_DIRTY; - -done: - /* Adjust cursors pointing to mp */ - mc->mc_pg[mc->mc_top] = np; - m2 = txn->mt_cursors[mc->mc_dbi]; - if (mc->mc_flags & C_SUB) { - for (; m2; m2=m2->mc_next) { - m3 = &m2->mc_xcursor->mx_cursor; - if (m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[mc->mc_top] == mp) - m3->mc_pg[mc->mc_top] = np; - } - } else { - for (; m2; m2=m2->mc_next) { - if (m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp) { - m2->mc_pg[mc->mc_top] = np; - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - IS_LEAF(np) && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - { - MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); - if (!(leaf->mn_flags & F_SUBDATA)) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - } - } - } - return 0; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_env_sync(MDB_env *env, int force) -{ - int rc = 0; - if (env->me_flags & MDB_RDONLY) - return EACCES; - if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { - if (env->me_flags & MDB_WRITEMAP) { - int flags = ((env->me_flags & MDB_MAPASYNC) && !force) - ? MS_ASYNC : MS_SYNC; - if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) - rc = ErrCode(); -#ifdef _WIN32 - else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); -#endif - } else { -#ifdef BROKEN_FDATASYNC - if (env->me_flags & MDB_FSYNCONLY) { - if (fsync(env->me_fd)) - rc = ErrCode(); - } else -#endif - if (MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); - } - } - return rc; -} - -/** Back up parent txn's cursors, then grab the originals for tracking */ -static int -mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) -{ - MDB_cursor *mc, *bk; - MDB_xcursor *mx; - size_t size; - int i; - - for (i = src->mt_numdbs; --i >= 0; ) { - if ((mc = src->mt_cursors[i]) != NULL) { - size = sizeof(MDB_cursor); - if (mc->mc_xcursor) - size += sizeof(MDB_xcursor); - for (; mc; mc = bk->mc_next) { - bk = malloc(size); - if (!bk) - return ENOMEM; - *bk = *mc; - mc->mc_backup = bk; - mc->mc_db = &dst->mt_dbs[i]; - /* Kill pointers into src - and dst to reduce abuse: The - * user may not use mc until dst ends. Otherwise we'd... - */ - mc->mc_txn = NULL; /* ...set this to dst */ - mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ - if ((mx = mc->mc_xcursor) != NULL) { - *(MDB_xcursor *)(bk+1) = *mx; - mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ - } - mc->mc_next = dst->mt_cursors[i]; - dst->mt_cursors[i] = mc; - } - } - } - return MDB_SUCCESS; -} - -/** Close this write txn's cursors, give parent txn's cursors back to parent. - * @param[in] txn the transaction handle. - * @param[in] merge true to keep changes to parent cursors, false to revert. - * @return 0 on success, non-zero on failure. - */ -static void -mdb_cursors_close(MDB_txn *txn, unsigned merge) -{ - MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; - MDB_xcursor *mx; - int i; - - for (i = txn->mt_numdbs; --i >= 0; ) { - for (mc = cursors[i]; mc; mc = next) { - next = mc->mc_next; - if ((bk = mc->mc_backup) != NULL) { - if (merge) { - /* Commit changes to parent txn */ - mc->mc_next = bk->mc_next; - mc->mc_backup = bk->mc_backup; - mc->mc_txn = bk->mc_txn; - mc->mc_db = bk->mc_db; - mc->mc_dbflag = bk->mc_dbflag; - if ((mx = mc->mc_xcursor) != NULL) - mx->mx_cursor.mc_txn = bk->mc_txn; - } else { - /* Abort nested txn */ - *mc = *bk; - if ((mx = mc->mc_xcursor) != NULL) - *mx = *(MDB_xcursor *)(bk+1); - } - mc = bk; - } - /* Only malloced cursors are permanently tracked. */ - free(mc); - } - cursors[i] = NULL; - } -} - -#if !(MDB_DEBUG) -#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) -#endif -static void -mdb_txn_reset0(MDB_txn *txn, const char *act); - -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ -enum Pidlock_op { - Pidset, Pidcheck -}; -#else -enum Pidlock_op { - Pidset = F_SETLK, Pidcheck = F_GETLK -}; -#endif - -/** Set or check a pid lock. Set returns 0 on success. - * Check returns 0 if the process is certainly dead, nonzero if it may - * be alive (the lock exists or an error happened so we do not know). - * - * On Windows Pidset is a no-op, we merely check for the existence - * of the process with the given pid. On POSIX we use a single byte - * lock on the lockfile, set at an offset equal to the pid. - */ -static int -mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) -{ -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ - int ret = 0; - HANDLE h; - if (op == Pidcheck) { - h = OpenProcess(env->me_pidquery, FALSE, pid); - /* No documented "no such process" code, but other program use this: */ - if (!h) - return ErrCode() != ERROR_INVALID_PARAMETER; - /* A process exists until all handles to it close. Has it exited? */ - ret = WaitForSingleObject(h, 0) != 0; - CloseHandle(h); - } - return ret; -#else - for (;;) { - int rc; - struct flock lock_info; - memset(&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = pid; - lock_info.l_len = 1; - if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { - if (op == F_GETLK && lock_info.l_type != F_UNLCK) - rc = -1; - } else if ((rc = ErrCode()) == EINTR) { - continue; - } - return rc; - } -#endif -} - -/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). - * @param[in] txn the transaction handle to initialize - * @return 0 on success, non-zero on failure. - */ -static int -mdb_txn_renew0(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_txninfo *ti = env->me_txns; - MDB_meta *meta; - unsigned int i, nr; - uint16_t x; - int rc, new_notls = 0; - - if (txn->mt_flags & MDB_TXN_RDONLY) { - txn->mt_flags = MDB_TXN_RDONLY; - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ - if (!ti) { - meta = env->me_metas[ mdb_env_pick_meta(env) ]; - txn->mt_txnid = meta->mm_txnid; - txn->mt_u.reader = NULL; - } else { - MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : - pthread_getspecific(env->me_txkey); - if (r) { - if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) - return MDB_BAD_RSLOT; - } else { - MDB_PID_T pid = env->me_pid; - MDB_THR_T tid = pthread_self(); - - if (!env->me_live_reader) { - rc = mdb_reader_pid(env, Pidset, pid); - if (rc) - return rc; - env->me_live_reader = 1; - } - - LOCK_MUTEX_R(env); - nr = ti->mti_numreaders; - for (i=0; imti_readers[i].mr_pid == 0) - break; - if (i == env->me_maxreaders) { - UNLOCK_MUTEX_R(env); - return MDB_READERS_FULL; - } - ti->mti_readers[i].mr_pid = pid; - ti->mti_readers[i].mr_tid = tid; - if (i == nr) - ti->mti_numreaders = ++nr; - /* Save numreaders for un-mutexed mdb_env_close() */ - env->me_numreaders = nr; - UNLOCK_MUTEX_R(env); - - r = &ti->mti_readers[i]; - new_notls = (env->me_flags & MDB_NOTLS); - if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { - r->mr_pid = 0; - return rc; - } - } - do /* LY: Retry on a race, ITS#7970. */ - r->mr_txnid = ti->mti_txnid; - while(r->mr_txnid != ti->mti_txnid); - txn->mt_txnid = r->mr_txnid; - txn->mt_u.reader = r; - meta = env->me_metas[txn->mt_txnid & 1]; - } - } else { - if (ti) { - LOCK_MUTEX_W(env); - - txn->mt_txnid = ti->mti_txnid; - meta = env->me_metas[txn->mt_txnid & 1]; - } else { - meta = env->me_metas[ mdb_env_pick_meta(env) ]; - txn->mt_txnid = meta->mm_txnid; - } - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - txn->mt_txnid++; -#if MDB_DEBUG - if (txn->mt_txnid == mdb_debug_start) - mdb_debug = 1; -#endif - txn->mt_flags = 0; - txn->mt_child = NULL; - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - txn->mt_dirty_room = MDB_IDL_UM_MAX; - txn->mt_u.dirty_list = env->me_dirty_list; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_free_pgs = env->me_free_pgs; - txn->mt_free_pgs[0] = 0; - txn->mt_spill_pgs = NULL; - env->me_txn = txn; - memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); - } - - /* Copy the DB info and flags */ - memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db)); - - /* Moved to here to avoid a data race in read TXNs */ - txn->mt_next_pgno = meta->mm_last_pg+1; - - for (i=2; imt_numdbs; i++) { - x = env->me_dbflags[i]; - txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; - txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; - } - txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; - - if (env->me_maxpg < txn->mt_next_pgno) { - mdb_txn_reset0(txn, "renew0-mapfail"); - if (new_notls) { - txn->mt_u.reader->mr_pid = 0; - txn->mt_u.reader = NULL; - } - return MDB_MAP_RESIZED; - } - - return MDB_SUCCESS; -} - -int -mdb_txn_renew(MDB_txn *txn) -{ - int rc; - - if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ - return EINVAL; - - if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { - DPUTS("environment had fatal error, must shutdown!"); - return MDB_PANIC; - } - - rc = mdb_txn_renew0(txn); - if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); - } - return rc; -} - -int -mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) -{ - MDB_txn *txn; - MDB_ntxn *ntxn; - int rc, size, tsize = sizeof(MDB_txn); - - if (env->me_flags & MDB_FATAL_ERROR) { - DPUTS("environment had fatal error, must shutdown!"); - return MDB_PANIC; - } - if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) - return EACCES; - if (parent) { - /* Nested transactions: Max 1 child, write txns only, no writemap */ - if (parent->mt_child || - (flags & MDB_RDONLY) || - (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || - (env->me_flags & MDB_WRITEMAP)) - { - return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; - } - tsize = sizeof(MDB_ntxn); - } - size = tsize; - if (!(flags & MDB_RDONLY)) { - if (!parent) { - txn = env->me_txn0; /* just reuse preallocated write txn */ - goto ok; - } - /* child txns use own copy of cursors */ - size += env->me_maxdbs * sizeof(MDB_cursor *); - } - size += env->me_maxdbs * (sizeof(MDB_db)+1); - - if ((txn = calloc(1, size)) == NULL) { - DPRINTF(("calloc: %s", strerror(errno))); - return ENOMEM; - } - txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); - if (flags & MDB_RDONLY) { - txn->mt_flags |= MDB_TXN_RDONLY; - txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = env->me_dbiseqs; - } else { - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - if (parent) { - txn->mt_dbiseqs = parent->mt_dbiseqs; - txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); - } else { - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - } - } - txn->mt_env = env; - -ok: - if (parent) { - unsigned int i; - txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); - if (!txn->mt_u.dirty_list || - !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) - { - free(txn->mt_u.dirty_list); - free(txn); - return ENOMEM; - } - txn->mt_txnid = parent->mt_txnid; - txn->mt_dirty_room = parent->mt_dirty_room; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_spill_pgs = NULL; - txn->mt_next_pgno = parent->mt_next_pgno; - parent->mt_child = txn; - txn->mt_parent = parent; - txn->mt_numdbs = parent->mt_numdbs; - txn->mt_flags = parent->mt_flags; - txn->mt_dbxs = parent->mt_dbxs; - memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - /* Copy parent's mt_dbflags, but clear DB_NEW */ - for (i=0; imt_numdbs; i++) - txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; - rc = 0; - ntxn = (MDB_ntxn *)txn; - ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ - if (env->me_pghead) { - size = MDB_IDL_SIZEOF(env->me_pghead); - env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); - if (env->me_pghead) - memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); - else - rc = ENOMEM; - } - if (!rc) - rc = mdb_cursor_shadow(parent, txn); - if (rc) - mdb_txn_reset0(txn, "beginchild-fail"); - } else { - rc = mdb_txn_renew0(txn); - } - if (rc) { - if (txn != env->me_txn0) - free(txn); - } else { - *ret = txn; - DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); - } - - return rc; -} - -MDB_env * -mdb_txn_env(MDB_txn *txn) -{ - if(!txn) return NULL; - return txn->mt_env; -} - -/** Export or close DBI handles opened in this txn. */ -static void -mdb_dbis_update(MDB_txn *txn, int keep) -{ - int i; - MDB_dbi n = txn->mt_numdbs; - MDB_env *env = txn->mt_env; - unsigned char *tdbflags = txn->mt_dbflags; - - for (i = n; --i >= 2;) { - if (tdbflags[i] & DB_NEW) { - if (keep) { - env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; - } else { - char *ptr = env->me_dbxs[i].md_name.mv_data; - if (ptr) { - env->me_dbxs[i].md_name.mv_data = NULL; - env->me_dbxs[i].md_name.mv_size = 0; - env->me_dbflags[i] = 0; - env->me_dbiseqs[i]++; - free(ptr); - } - } - } - } - if (keep && env->me_numdbs < n) - env->me_numdbs = n; -} - -/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). - * May be called twice for readonly txns: First reset it, then abort. - * @param[in] txn the transaction handle to reset - * @param[in] act why the transaction is being reset - */ -static void -mdb_txn_reset0(MDB_txn *txn, const char *act) -{ - MDB_env *env = txn->mt_env; - - /* Close any DBI handles opened in this txn */ - mdb_dbis_update(txn, 0); - - DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", - act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - if (txn->mt_u.reader) { - txn->mt_u.reader->mr_txnid = (txnid_t)-1; - if (!(env->me_flags & MDB_NOTLS)) - txn->mt_u.reader = NULL; /* txn does not own reader */ - } - txn->mt_numdbs = 0; /* close nothing if called again */ - txn->mt_dbxs = NULL; /* mark txn as reset */ - } else { - pgno_t *pghead = env->me_pghead; - - mdb_cursors_close(txn, 0); - if (!(env->me_flags & MDB_WRITEMAP)) { - mdb_dlist_free(txn); - } - - if (!txn->mt_parent) { - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; - /* me_pgstate: */ - env->me_pghead = NULL; - env->me_pglast = 0; - - env->me_txn = NULL; - /* The writer mutex was locked in mdb_txn_begin. */ - if (env->me_txns) - UNLOCK_MUTEX_W(env); - } else { - txn->mt_parent->mt_child = NULL; - env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; - mdb_midl_free(txn->mt_free_pgs); - mdb_midl_free(txn->mt_spill_pgs); - free(txn->mt_u.dirty_list); - } - - mdb_midl_free(pghead); - } -} - -void -mdb_txn_reset(MDB_txn *txn) -{ - if (txn == NULL) - return; - - /* This call is only valid for read-only txns */ - if (!(txn->mt_flags & MDB_TXN_RDONLY)) - return; - - mdb_txn_reset0(txn, "reset"); -} - -void -mdb_txn_abort(MDB_txn *txn) -{ - if (txn == NULL) - return; - - if (txn->mt_child) - mdb_txn_abort(txn->mt_child); - - mdb_txn_reset0(txn, "abort"); - /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ - if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) - txn->mt_u.reader->mr_pid = 0; - - if (txn != txn->mt_env->me_txn0) - free(txn); -} - -/** Save the freelist as of this transaction to the freeDB. - * This changes the freelist. Keep trying until it stabilizes. - */ -static int -mdb_freelist_save(MDB_txn *txn) -{ - /* env->me_pghead[] can grow and shrink during this call. - * env->me_pglast and txn->mt_free_pgs[] can only grow. - * Page numbers cannot disappear from txn->mt_free_pgs[]. - */ - MDB_cursor mc; - MDB_env *env = txn->mt_env; - int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; - txnid_t pglast = 0, head_id = 0; - pgno_t freecnt = 0, *free_pgs, *mop; - ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; - - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - - if (env->me_pghead) { - /* Make sure first page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (!env->me_pghead && txn->mt_loose_pgs) { - /* Put loose page numbers in mt_free_pgs, since - * we may be unable to return them to me_pghead. - */ - MDB_page *mp = txn->mt_loose_pgs; - if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) - return rc; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - } - - /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ - clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) - ? SSIZE_MAX : maxfree_1pg; - - for (;;) { - /* Come back here after each Put() in case freelist changed */ - MDB_val key, data; - pgno_t *pgs; - ssize_t j; - - /* If using records from freeDB which we have not yet - * deleted, delete them and any we reserved for me_pghead. - */ - while (pglast < env->me_pglast) { - rc = mdb_cursor_first(&mc, &key, NULL); - if (rc) - return rc; - pglast = head_id = *(txnid_t *)key.mv_data; - total_room = head_room = 0; - mdb_tassert(txn, pglast <= env->me_pglast); - rc = mdb_cursor_del(&mc, 0); - if (rc) - return rc; - } - - /* Save the IDL of pages freed by this txn, to a single record */ - if (freecnt < txn->mt_free_pgs[0]) { - if (!freecnt) { - /* Make sure last page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - free_pgs = txn->mt_free_pgs; - /* Write to last page of freeDB */ - key.mv_size = sizeof(txn->mt_txnid); - key.mv_data = &txn->mt_txnid; - do { - freecnt = free_pgs[0]; - data.mv_size = MDB_IDL_SIZEOF(free_pgs); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* Retry if mt_free_pgs[] grew during the Put() */ - free_pgs = txn->mt_free_pgs; - } while (freecnt < free_pgs[0]); - mdb_midl_sort(free_pgs); - memcpy(data.mv_data, free_pgs, data.mv_size); -#if (MDB_DEBUG) > 1 - { - unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", - txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); - for (; i; i--) - DPRINTF(("IDL %"Z"u", free_pgs[i])); - } -#endif - continue; - } - - mop = env->me_pghead; - mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; - - /* Reserve records for me_pghead[]. Split it if multi-page, - * to avoid searching freeDB for a page range. Use keys in - * range [1,me_pglast]: Smaller than txnid of oldest reader. - */ - if (total_room >= mop_len) { - if (total_room == mop_len || --more < 0) - break; - } else if (head_room >= maxfree_1pg && head_id > 1) { - /* Keep current record (overflow page), add a new one */ - head_id--; - head_room = 0; - } - /* (Re)write {key = head_id, IDL length = head_room} */ - total_room -= head_room; - head_room = mop_len - total_room; - if (head_room > maxfree_1pg && head_id > 1) { - /* Overflow multi-page for part of me_pghead */ - head_room /= head_id; /* amortize page sizes */ - head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); - } else if (head_room < 0) { - /* Rare case, not bothering to delete this record */ - head_room = 0; - } - key.mv_size = sizeof(head_id); - key.mv_data = &head_id; - data.mv_size = (head_room + 1) * sizeof(pgno_t); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* IDL is initially empty, zero out at least the length */ - pgs = (pgno_t *)data.mv_data; - j = head_room > clean_limit ? head_room : 0; - do { - pgs[j] = 0; - } while (--j >= 0); - total_room += head_room; - } - - /* Return loose page numbers to me_pghead, though usually none are - * left at this point. The pages themselves remain in dirty_list. - */ - if (txn->mt_loose_pgs) { - MDB_page *mp = txn->mt_loose_pgs; - unsigned count = txn->mt_loose_count; - MDB_IDL loose; - /* Room for loose pages + temp IDL with same */ - if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) - return rc; - mop = env->me_pghead; - loose = mop + MDB_IDL_ALLOCLEN(mop) - count; - for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - loose[ ++count ] = mp->mp_pgno; - loose[0] = count; - mdb_midl_sort(loose); - mdb_midl_xmerge(mop, loose); - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - mop_len = mop[0]; - } - - /* Fill in the reserved me_pghead records */ - rc = MDB_SUCCESS; - if (mop_len) { - MDB_val key, data; - - mop += mop_len; - rc = mdb_cursor_first(&mc, &key, &data); - for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { - txnid_t id = *(txnid_t *)key.mv_data; - ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; - MDB_ID save; - - mdb_tassert(txn, len >= 0 && id <= env->me_pglast); - key.mv_data = &id; - if (len > mop_len) { - len = mop_len; - data.mv_size = (len + 1) * sizeof(MDB_ID); - } - data.mv_data = mop -= len; - save = mop[0]; - mop[0] = len; - rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); - mop[0] = save; - if (rc || !(mop_len -= len)) - break; - } - } - return rc; -} - -/** Flush (some) dirty pages to the map, after clearing their dirty flag. - * @param[in] txn the transaction that's being committed - * @param[in] keep number of initial pages in dirty_list to keep dirty. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_flush(MDB_txn *txn, int keep) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned psize = env->me_psize, j; - int i, pagecount = dl[0].mid, rc; - size_t size = 0, pos = 0; - pgno_t pgno = 0; - MDB_page *dp = NULL; -#ifdef _WIN32 - OVERLAPPED ov; -#else - struct iovec iov[MDB_COMMIT_PAGES]; - ssize_t wpos = 0, wsize = 0, wres; - size_t next_pos = 1; /* impossible pos, so pos != next_pos */ - int n = 0; -#endif - - j = i = keep; - - if (env->me_flags & MDB_WRITEMAP) { - /* Clear dirty flags */ - while (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[++j] = dl[i]; - continue; - } - dp->mp_flags &= ~P_DIRTY; - } - goto done; - } - - /* Write the pages */ - for (;;) { - if (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[i].mid = 0; - continue; - } - pgno = dl[i].mid; - /* clear dirty flag */ - dp->mp_flags &= ~P_DIRTY; - pos = pgno * psize; - size = psize; - if (IS_OVERFLOW(dp)) size *= dp->mp_pages; - } -#ifdef _WIN32 - else break; - - /* Windows actually supports scatter/gather I/O, but only on - * unbuffered file handles. Since we're relying on the OS page - * cache for all our data, that's self-defeating. So we just - * write pages one at a time. We use the ov structure to set - * the write offset, to at least save the overhead of a Seek - * system call. - */ - DPRINTF(("committing page %"Z"u", pgno)); - memset(&ov, 0, sizeof(ov)); - ov.Offset = pos & 0xffffffff; - ov.OffsetHigh = pos >> 16 >> 16; - if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { - rc = ErrCode(); - DPRINTF(("WriteFile: %d", rc)); - return rc; - } -#else - /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ - if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { - if (n) { -retry_write: - /* Write previous page(s) */ -#ifdef MDB_USE_PWRITEV - wres = pwritev(env->me_fd, iov, n, wpos); -#else - if (n == 1) { - wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); - } else { -retry_seek: - if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { - rc = ErrCode(); - if (rc == EINTR) - goto retry_seek; - DPRINTF(("lseek: %s", strerror(rc))); - return rc; - } - wres = writev(env->me_fd, iov, n); - } -#endif - if (wres != wsize) { - if (wres < 0) { - rc = ErrCode(); - if (rc == EINTR) - goto retry_write; - DPRINTF(("Write error: %s", strerror(rc))); - } else { - rc = EIO; /* TODO: Use which error code? */ - DPUTS("short write, filesystem full?"); - } - return rc; - } - n = 0; - } - if (i > pagecount) - break; - wpos = pos; - wsize = 0; - } - DPRINTF(("committing page %"Z"u", pgno)); - next_pos = pos + size; - iov[n].iov_len = size; - iov[n].iov_base = (char *)dp; - wsize += size; - n++; -#endif /* _WIN32 */ - } - - /* MIPS has cache coherency issues, this is a no-op everywhere else - * Note: for any size >= on-chip cache size, entire on-chip cache is - * flushed. - */ - CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); - - for (i = keep; ++i <= pagecount; ) { - dp = dl[i].mptr; - /* This is a page we skipped above */ - if (!dl[i].mid) { - dl[++j] = dl[i]; - dl[j].mid = dp->mp_pgno; - continue; - } - mdb_dpage_free(env, dp); - } - -done: - i--; - txn->mt_dirty_room += i - j; - dl[0].mid = j; - return MDB_SUCCESS; -} - -int -mdb_txn_commit(MDB_txn *txn) -{ - int rc; - unsigned int i; - MDB_env *env; - - if (txn == NULL || txn->mt_env == NULL) - return EINVAL; - - if (txn->mt_child) { - rc = mdb_txn_commit(txn->mt_child); - txn->mt_child = NULL; - if (rc) - goto fail; - } - - env = txn->mt_env; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - mdb_dbis_update(txn, 1); - txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ - mdb_txn_abort(txn); - return MDB_SUCCESS; - } - - if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { - DPUTS("error flag is set, can't commit"); - if (txn->mt_parent) - txn->mt_parent->mt_flags |= MDB_TXN_ERROR; - rc = MDB_BAD_TXN; - goto fail; - } - - if (txn->mt_parent) { - MDB_txn *parent = txn->mt_parent; - MDB_page **lp; - MDB_ID2L dst, src; - MDB_IDL pspill; - unsigned x, y, len, ps_len; - - /* Append our free list to parent's */ - rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); - if (rc) - goto fail; - mdb_midl_free(txn->mt_free_pgs); - /* Failures after this must either undo the changes - * to the parent or set MDB_TXN_ERROR in the parent. - */ - - parent->mt_next_pgno = txn->mt_next_pgno; - parent->mt_flags = txn->mt_flags; - - /* Merge our cursors into parent's and close them */ - mdb_cursors_close(txn, 1); - - /* Update parent's DB table. */ - memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - parent->mt_numdbs = txn->mt_numdbs; - parent->mt_dbflags[0] = txn->mt_dbflags[0]; - parent->mt_dbflags[1] = txn->mt_dbflags[1]; - for (i=2; imt_numdbs; i++) { - /* preserve parent's DB_NEW status */ - x = parent->mt_dbflags[i] & DB_NEW; - parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; - } - - dst = parent->mt_u.dirty_list; - src = txn->mt_u.dirty_list; - /* Remove anything in our dirty list from parent's spill list */ - if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { - x = y = ps_len; - pspill[0] = (pgno_t)-1; - /* Mark our dirty pages as deleted in parent spill list */ - for (i=0, len=src[0].mid; ++i <= len; ) { - MDB_ID pn = src[i].mid << 1; - while (pn > pspill[x]) - x--; - if (pn == pspill[x]) { - pspill[x] = 1; - y = --x; - } - } - /* Squash deleted pagenums if we deleted any */ - for (x=y; ++x <= ps_len; ) - if (!(pspill[x] & 1)) - pspill[++y] = pspill[x]; - pspill[0] = y; - } - - /* Find len = length of merging our dirty list with parent's */ - x = dst[0].mid; - dst[0].mid = 0; /* simplify loops */ - if (parent->mt_parent) { - len = x + src[0].mid; - y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; - for (i = x; y && i; y--) { - pgno_t yp = src[y].mid; - while (yp < dst[i].mid) - i--; - if (yp == dst[i].mid) { - i--; - len--; - } - } - } else { /* Simplify the above for single-ancestor case */ - len = MDB_IDL_UM_MAX - txn->mt_dirty_room; - } - /* Merge our dirty list with parent's */ - y = src[0].mid; - for (i = len; y; dst[i--] = src[y--]) { - pgno_t yp = src[y].mid; - while (yp < dst[x].mid) - dst[i--] = dst[x--]; - if (yp == dst[x].mid) - free(dst[x--].mptr); - } - mdb_tassert(txn, i == x); - dst[0].mid = len; - free(txn->mt_u.dirty_list); - parent->mt_dirty_room = txn->mt_dirty_room; - if (txn->mt_spill_pgs) { - if (parent->mt_spill_pgs) { - /* TODO: Prevent failure here, so parent does not fail */ - rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); - if (rc) - parent->mt_flags |= MDB_TXN_ERROR; - mdb_midl_free(txn->mt_spill_pgs); - mdb_midl_sort(parent->mt_spill_pgs); - } else { - parent->mt_spill_pgs = txn->mt_spill_pgs; - } - } - - /* Append our loose page list to parent's */ - for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) - ; - *lp = txn->mt_loose_pgs; - parent->mt_loose_count += txn->mt_loose_count; - - parent->mt_child = NULL; - mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); - free(txn); - return rc; - } - - if (txn != env->me_txn) { - DPUTS("attempt to commit unknown transaction"); - rc = EINVAL; - goto fail; - } - - mdb_cursors_close(txn, 0); - - if (!txn->mt_u.dirty_list[0].mid && - !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) - goto done; - - DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", - txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); - - /* Update DB root pointers */ - if (txn->mt_numdbs > 2) { - MDB_cursor mc; - MDB_dbi i; - MDB_val data; - data.mv_size = sizeof(MDB_db); - - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - for (i = 2; i < txn->mt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - if (TXN_DBI_CHANGED(txn, i)) { - rc = MDB_BAD_DBI; - goto fail; - } - data.mv_data = &txn->mt_dbs[i]; - rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); - if (rc) - goto fail; - } - } - } - - rc = mdb_freelist_save(txn); - if (rc) - goto fail; - - mdb_midl_free(env->me_pghead); - env->me_pghead = NULL; - if (mdb_midl_shrink(&txn->mt_free_pgs)) - env->me_free_pgs = txn->mt_free_pgs; - -#if (MDB_DEBUG) > 2 - mdb_audit(txn); -#endif - - if ((rc = mdb_page_flush(txn, 0)) || - (rc = mdb_env_sync(env, 0)) || - (rc = mdb_env_write_meta(txn))) - goto fail; - - /* Free P_LOOSE pages left behind in dirty_list */ - if (!(env->me_flags & MDB_WRITEMAP)) - mdb_dlist_free(txn); - -done: - env->me_pglast = 0; - env->me_txn = NULL; - mdb_dbis_update(txn, 1); - - if (env->me_txns) - UNLOCK_MUTEX_W(env); - if (txn != env->me_txn0) - free(txn); - - return MDB_SUCCESS; - -fail: - mdb_txn_abort(txn); - return rc; -} - -/** Read the environment parameters of a DB environment before - * mapping it into memory. - * @param[in] env the environment handle - * @param[out] meta address of where to store the meta information - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_read_header(MDB_env *env, MDB_meta *meta) -{ - MDB_metabuf pbuf; - MDB_page *p; - MDB_meta *m; - int i, rc, off; - enum { Size = sizeof(pbuf) }; - - /* We don't know the page size yet, so use a minimum value. - * Read both meta pages so we can use the latest one. - */ - - for (i=off=0; i<2; i++, off = meta->mm_psize) { -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; - if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) - rc = 0; -#else - rc = pread(env->me_fd, &pbuf, Size, off); -#endif - if (rc != Size) { - if (rc == 0 && off == 0) - return ENOENT; - rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; - DPRINTF(("read: %s", mdb_strerror(rc))); - return rc; - } - - p = (MDB_page *)&pbuf; - - if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); - return MDB_INVALID; - } - - m = METADATA(p); - if (m->mm_magic != MDB_MAGIC) { - DPUTS("meta has invalid magic"); - return MDB_INVALID; - } - - if (m->mm_version != MDB_DATA_VERSION) { - DPRINTF(("database is version %u, expected version %u", - m->mm_version, MDB_DATA_VERSION)); - return MDB_VERSION_MISMATCH; - } - - if (off == 0 || m->mm_txnid > meta->mm_txnid) - *meta = *m; - } - return 0; -} - -static void ESECT -mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) -{ - meta->mm_magic = MDB_MAGIC; - meta->mm_version = MDB_DATA_VERSION; - meta->mm_mapsize = env->me_mapsize; - meta->mm_psize = env->me_psize; - meta->mm_last_pg = 1; - meta->mm_flags = env->me_flags & 0xffff; - meta->mm_flags |= MDB_INTEGERKEY; - meta->mm_dbs[0].md_root = P_INVALID; - meta->mm_dbs[1].md_root = P_INVALID; -} - -/** Write the environment parameters of a freshly created DB environment. - * @param[in] env the environment handle - * @param[out] meta address of where to store the meta information - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_init_meta(MDB_env *env, MDB_meta *meta) -{ - MDB_page *p, *q; - int rc; - unsigned int psize; -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - ov.Offset = pos; \ - rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) -#else - int len; -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - len = pwrite(fd, ptr, size, pos); \ - if (len == -1 && ErrCode() == EINTR) continue; \ - rc = (len >= 0); break; } while(1) -#endif - - DPUTS("writing new meta page"); - - psize = env->me_psize; - - mdb_env_init_meta0(env, meta); - - p = calloc(2, psize); - p->mp_pgno = 0; - p->mp_flags = P_META; - *(MDB_meta *)METADATA(p) = *meta; - - q = (MDB_page *)((char *)p + psize); - q->mp_pgno = 1; - q->mp_flags = P_META; - *(MDB_meta *)METADATA(q) = *meta; - - DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0); - if (!rc) - rc = ErrCode(); - else if ((unsigned) len == psize * 2) - rc = MDB_SUCCESS; - else - rc = ENOSPC; - free(p); - return rc; -} - -/** Update the environment info to commit a transaction. - * @param[in] txn the transaction that's being committed - * @return 0 on success, non-zero on failure. - */ -static int -mdb_env_write_meta(MDB_txn *txn) -{ - MDB_env *env; - MDB_meta meta, metab, *mp; - size_t mapsize; - off_t off; - int rc, len, toggle; - char *ptr; - HANDLE mfd; -#ifdef _WIN32 - OVERLAPPED ov; -#else - int r2; -#endif - - toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Z"u", - toggle, txn->mt_dbs[MAIN_DBI].md_root)); - - env = txn->mt_env; - mp = env->me_metas[toggle]; - mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; - /* Persist any increases of mapsize config */ - if (mapsize < env->me_mapsize) - mapsize = env->me_mapsize; - - if (env->me_flags & MDB_WRITEMAP) { - mp->mm_mapsize = mapsize; - mp->mm_dbs[0] = txn->mt_dbs[0]; - mp->mm_dbs[1] = txn->mt_dbs[1]; - mp->mm_last_pg = txn->mt_next_pgno - 1; - mp->mm_txnid = txn->mt_txnid; - if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { - unsigned meta_size = env->me_psize; - rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; - ptr = env->me_map; - if (toggle) { -#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ - if (meta_size < env->me_os_psize) - meta_size += meta_size; - else -#endif - ptr += meta_size; - } - if (MDB_MSYNC(ptr, meta_size, rc)) { - rc = ErrCode(); - goto fail; - } - } - goto done; - } - metab.mm_txnid = env->me_metas[toggle]->mm_txnid; - metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; - - meta.mm_mapsize = mapsize; - meta.mm_dbs[0] = txn->mt_dbs[0]; - meta.mm_dbs[1] = txn->mt_dbs[1]; - meta.mm_last_pg = txn->mt_next_pgno - 1; - meta.mm_txnid = txn->mt_txnid; - - off = offsetof(MDB_meta, mm_mapsize); - ptr = (char *)&meta + off; - len = sizeof(MDB_meta) - off; - if (toggle) - off += env->me_psize; - off += PAGEHDRSZ; - - /* Write to the SYNC fd */ - mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? - env->me_fd : env->me_mfd; -retry_write: -#ifdef _WIN32 - { - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) - rc = -1; - } -#else - rc = pwrite(mfd, ptr, len, off); -#endif - if (rc != len) { - rc = rc < 0 ? ErrCode() : EIO; - if (rc == EINTR) - goto retry_write; - DPUTS("write failed, disk error?"); - /* On a failure, the pagecache still contains the new data. - * Write some old data back, to prevent it from being used. - * Use the non-SYNC fd; we know it will fail anyway. - */ - meta.mm_last_pg = metab.mm_last_pg; - meta.mm_txnid = metab.mm_txnid; -#ifdef _WIN32 - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - WriteFile(env->me_fd, ptr, len, NULL, &ov); -#else - r2 = pwrite(env->me_fd, ptr, len, off); - (void)r2; /* Silence warnings. We don't care about pwrite's return value */ -#endif -fail: - env->me_flags |= MDB_FATAL_ERROR; - return rc; - } - /* MIPS has cache coherency issues, this is a no-op everywhere else */ - CACHEFLUSH(env->me_map + off, len, DCACHE); -done: - /* Memory ordering issues are irrelevant; since the entire writer - * is wrapped by wmutex, all of these changes will become visible - * after the wmutex is unlocked. Since the DB is multi-version, - * readers will get consistent data regardless of how fresh or - * how stale their view of these values is. - */ - if (env->me_txns) - env->me_txns->mti_txnid = txn->mt_txnid; - - return MDB_SUCCESS; -} - -/** Check both meta pages to see which one is newer. - * @param[in] env the environment handle - * @return meta toggle (0 or 1). - */ -static int -mdb_env_pick_meta(const MDB_env *env) -{ - return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); -} - -int ESECT -mdb_env_create(MDB_env **env) -{ - MDB_env *e; - - e = calloc(1, sizeof(MDB_env)); - if (!e) - return ENOMEM; - - e->me_maxreaders = DEFAULT_READERS; - e->me_maxdbs = e->me_numdbs = 2; - e->me_fd = INVALID_HANDLE_VALUE; - e->me_lfd = INVALID_HANDLE_VALUE; - e->me_mfd = INVALID_HANDLE_VALUE; -#ifdef MDB_USE_POSIX_SEM - e->me_rmutex = SEM_FAILED; - e->me_wmutex = SEM_FAILED; -#endif - e->me_pid = getpid(); - GET_PAGESIZE(e->me_os_psize); - VGMEMP_CREATE(e,0,0); - *env = e; - return MDB_SUCCESS; -} - -static int ESECT -mdb_env_map(MDB_env *env, void *addr) -{ - MDB_page *p; - unsigned int flags = env->me_flags; -#ifdef _WIN32 - int rc; - HANDLE mh; - LONG sizelo, sizehi; - size_t msize; - - if (flags & MDB_RDONLY) { - /* Don't set explicit map size, use whatever exists */ - msize = 0; - sizelo = 0; - sizehi = 0; - } else { - msize = env->me_mapsize; - sizelo = msize & 0xffffffff; - sizehi = msize >> 16 >> 16; /* only needed on Win64 */ - - /* Windows won't create mappings for zero length files. - * and won't map more than the file size. - * Just set the maxsize right now. - */ - if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo - || !SetEndOfFile(env->me_fd) - || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) - return ErrCode(); - } - - mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? - PAGE_READWRITE : PAGE_READONLY, - sizehi, sizelo, NULL); - if (!mh) - return ErrCode(); - env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? - FILE_MAP_WRITE : FILE_MAP_READ, - 0, 0, msize, addr); - rc = env->me_map ? 0 : ErrCode(); - CloseHandle(mh); - if (rc) - return rc; -#else - int prot = PROT_READ; - if (flags & MDB_WRITEMAP) { - prot |= PROT_WRITE; - if (ftruncate(env->me_fd, env->me_mapsize) < 0) - return ErrCode(); - } - env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, - env->me_fd, 0); - if (env->me_map == MAP_FAILED) { - env->me_map = NULL; - return ErrCode(); - } - - if (flags & MDB_NORDAHEAD) { - /* Turn off readahead. It's harmful when the DB is larger than RAM. */ -#ifdef MADV_RANDOM - madvise(env->me_map, env->me_mapsize, MADV_RANDOM); -#else -#ifdef POSIX_MADV_RANDOM - posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); -#endif /* POSIX_MADV_RANDOM */ -#endif /* MADV_RANDOM */ - } -#endif /* _WIN32 */ - - /* Can happen because the address argument to mmap() is just a - * hint. mmap() can pick another, e.g. if the range is in use. - * The MAP_FIXED flag would prevent that, but then mmap could - * instead unmap existing pages to make room for the new map. - */ - if (addr && env->me_map != addr) - return EBUSY; /* TODO: Make a new MDB_* error code? */ - - p = (MDB_page *)env->me_map; - env->me_metas[0] = METADATA(p); - env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_mapsize(MDB_env *env, size_t size) -{ - /* If env is already open, caller is responsible for making - * sure there are no active txns. - */ - if (env->me_map) { - int rc; - void *old; - if (env->me_txn) - return EINVAL; - if (!size) - size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; - else if (size < env->me_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; - if (size < minsize) - size = minsize; - } - munmap(env->me_map, env->me_mapsize); - env->me_mapsize = size; - old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; - rc = mdb_env_map(env, old); - if (rc) - return rc; - } - env->me_mapsize = size; - if (env->me_psize) - env->me_maxpg = env->me_mapsize / env->me_psize; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) -{ - if (env->me_map) - return EINVAL; - env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) -{ - if (env->me_map || readers < 1) - return EINVAL; - env->me_maxreaders = readers; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) -{ - if (!env || !readers) - return EINVAL; - *readers = env->me_maxreaders; - return MDB_SUCCESS; -} - -static int ESECT -mdb_fsize(HANDLE fd, size_t *size) -{ -#ifdef _WIN32 - LARGE_INTEGER fsize; - - if (!GetFileSizeEx(fd, &fsize)) - return ErrCode(); - - *size = fsize.QuadPart; -#else - struct stat st; - - if (fstat(fd, &st)) - return ErrCode(); - - *size = st.st_size; -#endif - return MDB_SUCCESS; -} - -#ifdef BROKEN_FDATASYNC -#include -#include -#endif - -/** Further setup required for opening an LMDB environment - */ -static int ESECT -mdb_env_open2(MDB_env *env) -{ - unsigned int flags = env->me_flags; - int i, newenv = 0, rc; - MDB_meta meta; - -#ifdef _WIN32 - /* See if we should use QueryLimited */ - rc = GetVersion(); - if ((rc & 0xff) > 5) - env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; - else - env->me_pidquery = PROCESS_QUERY_INFORMATION; -#endif /* _WIN32 */ -#ifdef BROKEN_FDATASYNC - /* ext3/ext4 fdatasync is broken on some older Linux kernels. - * https://lkml.org/lkml/2012/9/3/83 - * Kernels after 3.6-rc6 are known good. - * https://lkml.org/lkml/2012/9/10/556 - * See if the DB is on ext3/ext4, then check for new enough kernel - * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known - * to be patched. - */ - { - struct statfs st; - fstatfs(env->me_fd, &st); - while (st.f_type == 0xEF53) { - struct utsname uts; - int i; - uname(&uts); - if (uts.release[0] < '3') { - if (!strncmp(uts.release, "2.6.32.", 7)) { - i = atoi(uts.release+7); - if (i >= 60) - break; /* 2.6.32.60 and newer is OK */ - } else if (!strncmp(uts.release, "2.6.34.", 7)) { - i = atoi(uts.release+7); - if (i >= 15) - break; /* 2.6.34.15 and newer is OK */ - } - } else if (uts.release[0] == '3') { - i = atoi(uts.release+2); - if (i > 5) - break; /* 3.6 and newer is OK */ - if (i == 5) { - i = atoi(uts.release+4); - if (i >= 4) - break; /* 3.5.4 and newer is OK */ - } else if (i == 2) { - i = atoi(uts.release+4); - if (i >= 30) - break; /* 3.2.30 and newer is OK */ - } - } else { /* 4.x and newer is OK */ - break; - } - env->me_flags |= MDB_FSYNCONLY; - break; - } - } -#endif - - memset(&meta, 0, sizeof(meta)); - - if ((i = mdb_env_read_header(env, &meta)) != 0) { - if (i != ENOENT) - return i; - DPUTS("new mdbenv"); - newenv = 1; - env->me_psize = env->me_os_psize; - if (env->me_psize > MAX_PAGESIZE) - env->me_psize = MAX_PAGESIZE; - } else { - env->me_psize = meta.mm_psize; - } - - /* Was a mapsize configured? */ - if (!env->me_mapsize) { - /* If this is a new environment, take the default, - * else use the size recorded in the existing env. - */ - env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; - } else if (env->me_mapsize < meta.mm_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; - if (env->me_mapsize < minsize) - env->me_mapsize = minsize; - } - - rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); - if (rc) - return rc; - - if (newenv) { - if (flags & MDB_FIXEDMAP) - meta.mm_address = env->me_map; - i = mdb_env_init_meta(env, &meta); - if (i != MDB_SUCCESS) { - return i; - } - } - - env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; - env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) - - sizeof(indx_t); -#if !(MDB_MAXKEYSIZE) - env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); -#endif - env->me_maxpg = env->me_mapsize / env->me_psize; - -#if MDB_DEBUG - { - int toggle = mdb_env_pick_meta(env); - MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; - - DPRINTF(("opened database version %u, pagesize %u", - env->me_metas[0]->mm_version, env->me_psize)); - DPRINTF(("using meta page %d", toggle)); - DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Z"u", db->md_entries)); - DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); - DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); - DPRINTF(("root: %"Z"u", db->md_root)); - } -#endif - - return MDB_SUCCESS; -} - - -/** Release a reader thread's slot in the reader lock table. - * This function is called automatically when a thread exits. - * @param[in] ptr This points to the slot in the reader lock table. - */ -static void -mdb_env_reader_dest(void *ptr) -{ - MDB_reader *reader = ptr; - - reader->mr_pid = 0; -} - -#ifdef _WIN32 -/** Junk for arranging thread-specific callbacks on Windows. This is - * necessarily platform and compiler-specific. Windows supports up - * to 1088 keys. Let's assume nobody opens more than 64 environments - * in a single process, for now. They can override this if needed. - */ -#ifndef MAX_TLS_KEYS -#define MAX_TLS_KEYS 64 -#endif -static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; -static int mdb_tls_nkeys; - -static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) -{ - int i; - switch(reason) { - case DLL_PROCESS_ATTACH: break; - case DLL_THREAD_ATTACH: break; - case DLL_THREAD_DETACH: - for (i=0; ime_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; - -#ifdef _WIN32 - { - OVERLAPPED ov; - /* First acquire a shared lock. The Unlock will - * then release the existing exclusive lock. - */ - memset(&ov, 0, sizeof(ov)); - if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { - rc = ErrCode(); - } else { - UnlockFile(env->me_lfd, 0, 0, 1, 0); - *excl = 0; - } - } -#else - { - struct flock lock_info; - /* The shared lock replaces the existing lock */ - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_RDLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - *excl = rc ? -1 : 0; /* error may mean we lost the lock */ - } -#endif - - return rc; -} - -/** Try to get exclusive lock, otherwise shared. - * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. - */ -static int ESECT -mdb_env_excl_lock(MDB_env *env, int *excl) -{ - int rc = 0; -#ifdef _WIN32 - if (LockFile(env->me_lfd, 0, 0, 1, 0)) { - *excl = 1; - } else { - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { - *excl = 0; - } else { - rc = ErrCode(); - } - } -#else - struct flock lock_info; - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (!rc) { - *excl = 1; - } else -# ifdef MDB_USE_POSIX_SEM - if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ -# endif - { - lock_info.l_type = F_RDLCK; - while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (rc == 0) - *excl = 0; - } -#endif - return rc; -} - -#ifdef MDB_USE_HASH -/* - * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code - * - * @(#) $Revision: 5.1 $ - * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ - * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ - * - * http://www.isthe.com/chongo/tech/comp/fnv/index.html - * - *** - * - * Please do not copyright this code. This code is in the public domain. - * - * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, - * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO - * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR - * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF - * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR - * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - * PERFORMANCE OF THIS SOFTWARE. - * - * By: - * chongo /\oo/\ - * http://www.isthe.com/chongo/ - * - * Share and Enjoy! :-) - */ - -typedef unsigned long long mdb_hash_t; -#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) - -/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer - * @param[in] val value to hash - * @param[in] hval initial value for hash - * @return 64 bit hash - * - * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the - * hval arg on the first call. - */ -static mdb_hash_t -mdb_hash_val(MDB_val *val, mdb_hash_t hval) -{ - unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ - unsigned char *end = s + val->mv_size; - /* - * FNV-1a hash each octet of the string - */ - while (s < end) { - /* xor the bottom with the current octet */ - hval ^= (mdb_hash_t)*s++; - - /* multiply by the 64 bit FNV magic prime mod 2^64 */ - hval += (hval << 1) + (hval << 4) + (hval << 5) + - (hval << 7) + (hval << 8) + (hval << 40); - } - /* return our new hash value */ - return hval; -} - -/** Hash the string and output the encoded hash. - * This uses modified RFC1924 Ascii85 encoding to accommodate systems with - * very short name limits. We don't care about the encoding being reversible, - * we just want to preserve as many bits of the input as possible in a - * small printable string. - * @param[in] str string to hash - * @param[out] encbuf an array of 11 chars to hold the hash - */ -static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; - -static void -mdb_pack85(unsigned long l, char *out) -{ - int i; - - for (i=0; i<5; i++) { - *out++ = mdb_a85[l % 85]; - l /= 85; - } -} - -static void -mdb_hash_enc(MDB_val *val, char *encbuf) -{ - mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); - - mdb_pack85(h, encbuf); - mdb_pack85(h>>32, encbuf+5); - encbuf[10] = '\0'; -} -#endif - -/** Open and/or initialize the lock region for the environment. - * @param[in] env The LMDB environment. - * @param[in] lpath The pathname of the file used for the lock region. - * @param[in] mode The Unix permissions for the file, if we create it. - * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) -{ -#ifdef _WIN32 -# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT -#else -# define MDB_ERRCODE_ROFS EROFS -#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ -# define MDB_CLOEXEC O_CLOEXEC -#else - int fdflags; -# define MDB_CLOEXEC 0 -#endif -#endif - int rc; - off_t size, rsize; - -#ifdef _WIN32 - env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL, NULL); -#else - env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); -#endif - if (env->me_lfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { - return MDB_SUCCESS; - } - goto fail_errno; - } -#if ! ((MDB_CLOEXEC) || defined(_WIN32)) - /* Lose record locks when exec*() */ - if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) - fcntl(env->me_lfd, F_SETFD, fdflags); -#endif - - if (!(env->me_flags & MDB_NOTLS)) { - rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); - if (rc) - goto fail; - env->me_flags |= MDB_ENV_TXKEY; -#ifdef _WIN32 - /* Windows TLS callbacks need help finding their TLS info. */ - if (mdb_tls_nkeys >= MAX_TLS_KEYS) { - rc = MDB_TLS_FULL; - goto fail; - } - mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; -#endif - } - - /* Try to get exclusive lock. If we succeed, then - * nobody is using the lock region and we should initialize it. - */ - if ((rc = mdb_env_excl_lock(env, excl))) goto fail; - -#ifdef _WIN32 - size = GetFileSize(env->me_lfd, NULL); -#else - size = lseek(env->me_lfd, 0, SEEK_END); - if (size == -1) goto fail_errno; -#endif - rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); - if (size < rsize && *excl > 0) { -#ifdef _WIN32 - if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize - || !SetEndOfFile(env->me_lfd)) - goto fail_errno; -#else - if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; -#endif - } else { - rsize = size; - size = rsize - sizeof(MDB_txninfo); - env->me_maxreaders = size/sizeof(MDB_reader) + 1; - } - { -#ifdef _WIN32 - HANDLE mh; - mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, - 0, 0, NULL); - if (!mh) goto fail_errno; - env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); - CloseHandle(mh); - if (!env->me_txns) goto fail_errno; -#else - void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, - env->me_lfd, 0); - if (m == MAP_FAILED) goto fail_errno; - env->me_txns = m; -#endif - } - if (*excl > 0) { -#ifdef _WIN32 - BY_HANDLE_FILE_INFORMATION stbuf; - struct { - DWORD volume; - DWORD nhigh; - DWORD nlow; - } idbuf; - MDB_val val; - char encbuf[11]; - - if (!mdb_sec_inited) { - InitializeSecurityDescriptor(&mdb_null_sd, - SECURITY_DESCRIPTOR_REVISION); - SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); - mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); - mdb_all_sa.bInheritHandle = FALSE; - mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; - mdb_sec_inited = 1; - } - if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; - idbuf.volume = stbuf.dwVolumeSerialNumber; - idbuf.nhigh = stbuf.nFileIndexHigh; - idbuf.nlow = stbuf.nFileIndexLow; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); - sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); - env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); - if (!env->me_wmutex) goto fail_errno; -#elif defined(MDB_USE_POSIX_SEM) - struct stat stbuf; - struct { - dev_t dev; - ino_t ino; - } idbuf; - MDB_val val; - char encbuf[11]; - -#if defined(__NetBSD__) -#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ -#endif - if (fstat(env->me_lfd, &stbuf)) goto fail_errno; - idbuf.dev = stbuf.st_dev; - idbuf.ino = stbuf.st_ino; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); -#ifdef MDB_SHORT_SEMNAMES - encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ -#endif - sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); - /* Clean up after a previous run, if needed: Try to - * remove both semaphores before doing anything else. - */ - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); - env->me_rmutex = sem_open(env->me_txns->mti_rmname, - O_CREAT|O_EXCL, mode, 1); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, - O_CREAT|O_EXCL, mode, 1); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#else /* MDB_USE_POSIX_SEM */ - pthread_mutexattr_t mattr; - - if ((rc = pthread_mutexattr_init(&mattr)) - || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) - || (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr)) - || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) - goto fail; - pthread_mutexattr_destroy(&mattr); -#endif /* _WIN32 || MDB_USE_POSIX_SEM */ - - env->me_txns->mti_magic = MDB_MAGIC; - env->me_txns->mti_format = MDB_LOCK_FORMAT; - env->me_txns->mti_txnid = 0; - env->me_txns->mti_numreaders = 0; - - } else { - if (env->me_txns->mti_magic != MDB_MAGIC) { - DPUTS("lock region has invalid magic"); - rc = MDB_INVALID; - goto fail; - } - if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { - DPRINTF(("lock region has format+version 0x%x, expected 0x%x", - env->me_txns->mti_format, MDB_LOCK_FORMAT)); - rc = MDB_VERSION_MISMATCH; - goto fail; - } - rc = ErrCode(); - if (rc && rc != EACCES && rc != EAGAIN) { - goto fail; - } -#ifdef _WIN32 - env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); - if (!env->me_wmutex) goto fail_errno; -#elif defined(MDB_USE_POSIX_SEM) - env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#endif - } - return MDB_SUCCESS; - -fail_errno: - rc = ErrCode(); -fail: - return rc; -} - - /** The name of the lock file in the DB environment */ -#define LOCKNAME "/lock.mdb" - /** The name of the data file in the DB environment */ -#define DATANAME "/data.mdb" - /** The suffix of the lock file when no subdir is used */ -#define LOCKSUFF "-lock" - /** Only a subset of the @ref mdb_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. - */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) -#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ - MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) - -#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) -# error "Persistent DB flags & env flags overlap, but both go in mm_flags" -#endif - -int ESECT -mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) -{ - int oflags, rc, len, excl = -1; - char *lpath, *dpath; - - if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) - return EINVAL; - - len = strlen(path); - if (flags & MDB_NOSUBDIR) { - rc = len + sizeof(LOCKSUFF) + len + 1; - } else { - rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); - } - lpath = malloc(rc); - if (!lpath) - return ENOMEM; - if (flags & MDB_NOSUBDIR) { - dpath = lpath + len + sizeof(LOCKSUFF); - sprintf(lpath, "%s" LOCKSUFF, path); - strcpy(dpath, path); - } else { - dpath = lpath + len + sizeof(LOCKNAME); - sprintf(lpath, "%s" LOCKNAME, path); - sprintf(dpath, "%s" DATANAME, path); - } - - rc = MDB_SUCCESS; - flags |= env->me_flags; - if (flags & MDB_RDONLY) { - /* silently ignore WRITEMAP when we're only getting read access */ - flags &= ~MDB_WRITEMAP; - } else { - if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && - (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) - rc = ENOMEM; - } - env->me_flags = flags |= MDB_ENV_ACTIVE; - if (rc) - goto leave; - - env->me_path = strdup(path); - env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); - env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); - env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); - if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { - rc = ENOMEM; - goto leave; - } - - /* For RDONLY, get lockfile after we know datafile exists */ - if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); - if (rc) - goto leave; - } - -#ifdef _WIN32 - if (F_ISSET(flags, MDB_RDONLY)) { - oflags = GENERIC_READ; - len = OPEN_EXISTING; - } else { - oflags = GENERIC_READ|GENERIC_WRITE; - len = OPEN_ALWAYS; - } - mode = FILE_ATTRIBUTE_NORMAL; - env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, - NULL, len, mode, NULL); -#else - if (F_ISSET(flags, MDB_RDONLY)) - oflags = O_RDONLY; - else - oflags = O_RDWR | O_CREAT; - - env->me_fd = open(dpath, oflags, mode); -#endif - if (env->me_fd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); - if (rc) - goto leave; - } - - if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { - if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { - env->me_mfd = env->me_fd; - } else { - /* Synchronous fd for meta writes. Needed even with - * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. - */ -#ifdef _WIN32 - len = OPEN_EXISTING; - env->me_mfd = CreateFile(dpath, oflags, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, - mode | FILE_FLAG_WRITE_THROUGH, NULL); -#else - oflags &= ~O_CREAT; - env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); -#endif - if (env->me_mfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - } - DPRINTF(("opened dbenv %p", (void *) env)); - if (excl > 0) { - rc = mdb_env_share_locks(env, &excl); - if (rc) - goto leave; - } - if (!((flags & MDB_RDONLY) || - (env->me_pbuf = calloc(1, env->me_psize)))) - rc = ENOMEM; - if (!(flags & MDB_RDONLY)) { - MDB_txn *txn; - int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * - (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); - txn = calloc(1, size); - if (txn) { - txn->mt_dbs = (MDB_db *)((char *)txn + tsize); - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - txn->mt_env = env; - txn->mt_dbxs = env->me_dbxs; - env->me_txn0 = txn; - } else { - rc = ENOMEM; - } - } - } - -leave: - if (rc) { - mdb_env_close0(env, excl); - } - free(lpath); - return rc; -} - -/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ -static void ESECT -mdb_env_close0(MDB_env *env, int excl) -{ - int i; - - if (!(env->me_flags & MDB_ENV_ACTIVE)) - return; - - /* Doing this here since me_dbxs may not exist during mdb_env_close */ - if (env->me_dbxs) { - for (i = env->me_maxdbs; --i > MAIN_DBI; ) - free(env->me_dbxs[i].md_name.mv_data); - free(env->me_dbxs); - } - - free(env->me_pbuf); - free(env->me_dbiseqs); - free(env->me_dbflags); - free(env->me_path); - free(env->me_dirty_list); - free(env->me_txn0); - mdb_midl_free(env->me_free_pgs); - - if (env->me_flags & MDB_ENV_TXKEY) { - pthread_key_delete(env->me_txkey); -#ifdef _WIN32 - /* Delete our key from the global list */ - for (i=0; ime_txkey) { - mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; - mdb_tls_nkeys--; - break; - } -#endif - } - - if (env->me_map) { - munmap(env->me_map, env->me_mapsize); - } - if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) - (void) close(env->me_mfd); - if (env->me_fd != INVALID_HANDLE_VALUE) - (void) close(env->me_fd); - if (env->me_txns) { - MDB_PID_T pid = env->me_pid; - /* Clearing readers is done in this function because - * me_txkey with its destructor must be disabled first. - */ - for (i = env->me_numreaders; --i >= 0; ) - if (env->me_txns->mti_readers[i].mr_pid == pid) - env->me_txns->mti_readers[i].mr_pid = 0; -#ifdef _WIN32 - if (env->me_rmutex) { - CloseHandle(env->me_rmutex); - if (env->me_wmutex) CloseHandle(env->me_wmutex); - } - /* Windows automatically destroys the mutexes when - * the last handle closes. - */ -#elif defined(MDB_USE_POSIX_SEM) - if (env->me_rmutex != SEM_FAILED) { - sem_close(env->me_rmutex); - if (env->me_wmutex != SEM_FAILED) - sem_close(env->me_wmutex); - /* If we have the filelock: If we are the - * only remaining user, clean up semaphores. - */ - if (excl == 0) - mdb_env_excl_lock(env, &excl); - if (excl > 0) { - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); - } - } -#endif - munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); - } - if (env->me_lfd != INVALID_HANDLE_VALUE) { -#ifdef _WIN32 - if (excl >= 0) { - /* Unlock the lockfile. Windows would have unlocked it - * after closing anyway, but not necessarily at once. - */ - UnlockFile(env->me_lfd, 0, 0, 1, 0); - } -#endif - (void) close(env->me_lfd); - } - - env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); -} - - -void ESECT -mdb_env_close(MDB_env *env) -{ - MDB_page *dp; - - if (env == NULL) - return; - - VGMEMP_DESTROY(env); - while ((dp = env->me_dpages) != NULL) { - VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); - env->me_dpages = dp->mp_next; - free(dp); - } - - mdb_env_close0(env, 0); - free(env); -} - -/** Compare two items pointing at aligned size_t's */ -static int -mdb_cmp_long(const MDB_val *a, const MDB_val *b) -{ - return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : - *(size_t *)a->mv_data > *(size_t *)b->mv_data; -} - -/** Compare two items pointing at aligned unsigned int's */ -static int -mdb_cmp_int(const MDB_val *a, const MDB_val *b) -{ - return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : - *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; -} - -/** Compare two items pointing at unsigned ints of unknown alignment. - * Nodes and keys are guaranteed to be 2-byte aligned. - */ -static int -mdb_cmp_cint(const MDB_val *a, const MDB_val *b) -{ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short *u, *c; - int x; - - u = (unsigned short *) ((char *) a->mv_data + a->mv_size); - c = (unsigned short *) ((char *) b->mv_data + a->mv_size); - do { - x = *--u - *--c; - } while(!x && u > (unsigned short *)a->mv_data); - return x; -#else - unsigned short *u, *c, *end; - int x; - - end = (unsigned short *) ((char *) a->mv_data + a->mv_size); - u = (unsigned short *)a->mv_data; - c = (unsigned short *)b->mv_data; - do { - x = *u++ - *c++; - } while(!x && u < end); - return x; -#endif -} - -/** Compare two items pointing at size_t's of unknown alignment. */ -#ifdef MISALIGNED_OK -# define mdb_cmp_clong mdb_cmp_long -#else -# define mdb_cmp_clong mdb_cmp_cint -#endif - -/** Compare two items lexically */ -static int -mdb_cmp_memn(const MDB_val *a, const MDB_val *b) -{ - int diff; - ssize_t len_diff; - unsigned int len; - - len = a->mv_size; - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - len = b->mv_size; - len_diff = 1; - } - - diff = memcmp(a->mv_data, b->mv_data, len); - return diff ? diff : len_diff<0 ? -1 : len_diff; -} - -/** Compare two items in reverse byte order */ -static int -mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) -{ - const unsigned char *p1, *p2, *p1_lim; - ssize_t len_diff; - int diff; - - p1_lim = (const unsigned char *)a->mv_data; - p1 = (const unsigned char *)a->mv_data + a->mv_size; - p2 = (const unsigned char *)b->mv_data + b->mv_size; - - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - p1_lim += len_diff; - len_diff = 1; - } - - while (p1 > p1_lim) { - diff = *--p1 - *--p2; - if (diff) - return diff; - } - return len_diff<0 ? -1 : len_diff; -} - -/** Search for key within a page, using binary search. - * Returns the smallest entry larger or equal to the key. - * If exactp is non-null, stores whether the found entry was an exact match - * in *exactp (1 or 0). - * Updates the cursor index with the index of the found entry. - * If no entry larger or equal to the key is found, returns NULL. - */ -static MDB_node * -mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) -{ - unsigned int i = 0, nkeys; - int low, high; - int rc = 0; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NULL; - MDB_val nodekey; - MDB_cmp_func *cmp; - DKBUF; - - nkeys = NUMKEYS(mp); - - DPRINTF(("searching %u keys in %s %spage %"Z"u", - nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp))); - - low = IS_LEAF(mp) ? 0 : 1; - high = nkeys - 1; - cmp = mc->mc_dbx->md_cmp; - - /* Branch pages have no data, so if using integer keys, - * alignment is guaranteed. Use faster mdb_cmp_int. - */ - if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { - if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) - cmp = mdb_cmp_long; - else - cmp = mdb_cmp_int; - } - - if (IS_LEAF2(mp)) { - nodekey.mv_size = mc->mc_db->md_pad; - node = NODEPTR(mp, 0); /* fake */ - while (low <= high) { - i = (low + high) >> 1; - nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); - rc = cmp(key, &nodekey); - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } else { - while (low <= high) { - i = (low + high) >> 1; - - node = NODEPTR(mp, i); - nodekey.mv_size = NODEKSZ(node); - nodekey.mv_data = NODEKEY(node); - - rc = cmp(key, &nodekey); -#if MDB_DEBUG - if (IS_LEAF(mp)) - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - else - DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", - i, DKEY(&nodekey), NODEPGNO(node), rc)); -#endif - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } - - if (rc > 0) { /* Found entry is less than the key. */ - i++; /* Skip to get the smallest entry larger than key. */ - if (!IS_LEAF2(mp)) - node = NODEPTR(mp, i); - } - if (exactp) - *exactp = (rc == 0 && nkeys > 0); - /* store the key index */ - mc->mc_ki[mc->mc_top] = i; - if (i >= nkeys) - /* There is no entry larger or equal to the key. */ - return NULL; - - /* nodeptr is fake for LEAF2 */ - return node; -} - -#if 0 -static void -mdb_cursor_adjust(MDB_cursor *mc, func) -{ - MDB_cursor *m2; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { - func(mc, m2); - } - } -} -#endif - -/** Pop a page off the top of the cursor's stack. */ -static void -mdb_cursor_pop(MDB_cursor *mc) -{ - if (mc->mc_snum) { -#if MDB_DEBUG - MDB_page *top = mc->mc_pg[mc->mc_top]; -#endif - mc->mc_snum--; - if (mc->mc_snum) - mc->mc_top--; - - DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, - DDBI(mc), (void *) mc)); - } -} - -/** Push a page onto the top of the cursor's stack. */ -static int -mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) -{ - DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, - DDBI(mc), (void *) mc)); - - if (mc->mc_snum >= CURSOR_STACK) { - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CURSOR_FULL; - } - - mc->mc_top = mc->mc_snum++; - mc->mc_pg[mc->mc_top] = mp; - mc->mc_ki[mc->mc_top] = 0; - - return MDB_SUCCESS; -} - -/** Find the address of the page corresponding to a given page number. - * @param[in] txn the transaction for this access. - * @param[in] pgno the page number for the page to retrieve. - * @param[out] ret address of a pointer where the page's address will be stored. - * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) -{ - MDB_env *env = txn->mt_env; - MDB_page *p = NULL; - int level; - - if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { - MDB_txn *tx2 = txn; - level = 1; - do { - MDB_ID2L dl = tx2->mt_u.dirty_list; - unsigned x; - /* Spilled pages were dirtied in this txn and flushed - * because the dirty list got full. Bring this page - * back in from the map (but don't unspill it here, - * leave that unless page_touch happens again). - */ - if (tx2->mt_spill_pgs) { - MDB_ID pn = pgno << 1; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - goto done; - } - } - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - p = dl[x].mptr; - goto done; - } - } - level++; - } while ((tx2 = tx2->mt_parent) != NULL); - } - - if (pgno < txn->mt_next_pgno) { - level = 0; - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - } else { - DPRINTF(("page %"Z"u not found", pgno)); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_NOTFOUND; - } - -done: - *ret = p; - if (lvl) - *lvl = level; - return MDB_SUCCESS; -} - -/** Finish #mdb_page_search() / #mdb_page_search_lowest(). - * The cursor is at the root page, set up the rest of it. - */ -static int -mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int rc; - DKBUF; - - while (IS_BRANCH(mp)) { - MDB_node *node; - indx_t i; - - DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); - mdb_cassert(mc, NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); - - if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { - i = 0; - if (flags & MDB_PS_LAST) - i = NUMKEYS(mp) - 1; - } else { - int exact; - node = mdb_node_search(mc, key, &exact); - if (node == NULL) - i = NUMKEYS(mp) - 1; - else { - i = mc->mc_ki[mc->mc_top]; - if (!exact) { - mdb_cassert(mc, i > 0); - i--; - } - } - DPRINTF(("following index %u for key [%s]", i, DKEY(key))); - } - - mdb_cassert(mc, i < NUMKEYS(mp)); - node = NODEPTR(mp, i); - - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = i; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc)) != 0) - return rc; - mp = mc->mc_pg[mc->mc_top]; - } - } - - if (!IS_LEAF(mp)) { - DPRINTF(("internal error, index points to a %02X page!?", - mp->mp_flags)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - - DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, - key ? DKEY(key) : "null")); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - return MDB_SUCCESS; -} - -/** Search for the lowest key under the current branch page. - * This just bypasses a NUMKEYS check in the current page - * before calling mdb_page_search_root(), because the callers - * are all in situations where the current page is known to - * be underfilled. - */ -static int -mdb_page_search_lowest(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NODEPTR(mp, 0); - int rc; - - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = 0; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); -} - -/** Search for the page a given key should be in. - * Push it and its parent pages on the cursor stack. - * @param[in,out] mc the cursor for this operation. - * @param[in] key the key to search for, or NULL for first/last page. - * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB - * are touched (updated with new page numbers). - * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. - * This is used by #mdb_cursor_first() and #mdb_cursor_last(). - * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) -{ - int rc; - pgno_t root; - - /* Make sure the txn is still viable, then find the root from - * the txn's db table and set it as the root of the cursor's stack. - */ - if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { - DPUTS("transaction has failed, must abort"); - return MDB_BAD_TXN; - } else { - /* Make sure we're using an up-to-date root */ - if (*mc->mc_dbflag & DB_STALE) { - MDB_cursor mc2; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); - if (rc) - return rc; - { - MDB_val data; - int exact = 0; - uint16_t flags; - MDB_node *leaf = mdb_node_search(&mc2, - &mc->mc_dbx->md_name, &exact); - if (!exact) - return MDB_NOTFOUND; - rc = mdb_node_read(mc->mc_txn, leaf, &data); - if (rc) - return rc; - memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), - sizeof(uint16_t)); - /* The txn may not know this DBI, or another process may - * have dropped and recreated the DB with other flags. - */ - if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) - return MDB_INCOMPATIBLE; - memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); - } - *mc->mc_dbflag &= ~DB_STALE; - } - root = mc->mc_db->md_root; - - if (root == P_INVALID) { /* Tree is empty. */ - DPUTS("tree is empty"); - return MDB_NOTFOUND; - } - } - - mdb_cassert(mc, root > 1); - if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) - if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) - return rc; - - mc->mc_snum = 1; - mc->mc_top = 0; - - DPRINTF(("db %d root page %"Z"u has flags 0x%X", - DDBI(mc), root, mc->mc_pg[0]->mp_flags)); - - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc))) - return rc; - } - - if (flags & MDB_PS_ROOTONLY) - return MDB_SUCCESS; - - return mdb_page_search_root(mc, key, flags); -} - -static int -mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) -{ - MDB_txn *txn = mc->mc_txn; - pgno_t pg = mp->mp_pgno; - unsigned x = 0, ovpages = mp->mp_pages; - MDB_env *env = txn->mt_env; - MDB_IDL sl = txn->mt_spill_pgs; - MDB_ID pn = pg << 1; - int rc; - - DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); - /* If the page is dirty or on the spill list we just acquired it, - * so we should give it back to our current free list, if any. - * Otherwise put it onto the list of pages we freed in this txn. - * - * Won't create me_pghead: me_pglast must be inited along with it. - * Unsupported in nested txns: They would need to hide the page - * range in ancestor txns' dirty and spilled lists. - */ - if (env->me_pghead && - !txn->mt_parent && - ((mp->mp_flags & P_DIRTY) || - (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) - { - unsigned i, j; - pgno_t *mop; - MDB_ID2 *dl, ix, iy; - rc = mdb_midl_need(&env->me_pghead, ovpages); - if (rc) - return rc; - if (!(mp->mp_flags & P_DIRTY)) { - /* This page is no longer spilled */ - if (x == sl[0]) - sl[0]--; - else - sl[x] |= 1; - goto release; - } - /* Remove from dirty list */ - dl = txn->mt_u.dirty_list; - x = dl[0].mid--; - for (ix = dl[x]; ix.mptr != mp; ix = iy) { - if (x > 1) { - x--; - iy = dl[x]; - dl[x] = ix; - } else { - mdb_cassert(mc, x > 1); - j = ++(dl[0].mid); - dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - } - if (!(env->me_flags & MDB_WRITEMAP)) - mdb_dpage_free(env, mp); -release: - /* Insert in me_pghead */ - mop = env->me_pghead; - j = mop[0] + ovpages; - for (i = mop[0]; i && mop[i] < pg; i--) - mop[j--] = mop[i]; - while (j>i) - mop[j--] = pg++; - mop[0] += ovpages; - } else { - rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); - if (rc) - return rc; - } - mc->mc_db->md_overflow_pages -= ovpages; - return 0; -} - -/** Return the data associated with a given node. - * @param[in] txn The transaction for this operation. - * @param[in] leaf The node being read. - * @param[out] data Updated to point to the node's data. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) -{ - MDB_page *omp; /* overflow page */ - pgno_t pgno; - int rc; - - if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { - data->mv_size = NODEDSZ(leaf); - data->mv_data = NODEDATA(leaf); - return MDB_SUCCESS; - } - - /* Read overflow data. - */ - data->mv_size = NODEDSZ(leaf); - memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); - if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Z"u failed", pgno)); - return rc; - } - data->mv_data = METADATA(omp); - - return MDB_SUCCESS; -} - -int -mdb_get(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - MDB_cursor mc; - MDB_xcursor mx; - int exact = 0; - DKBUF; - - DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); - - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - mdb_cursor_init(&mc, txn, dbi, &mx); - return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); -} - -/** Find a sibling for a page. - * Replaces the page at the top of the cursor's stack with the - * specified sibling, if one exists. - * @param[in] mc The cursor for this operation. - * @param[in] move_right Non-zero if the right sibling is requested, - * otherwise the left sibling. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_cursor_sibling(MDB_cursor *mc, int move_right) -{ - int rc; - MDB_node *indx; - MDB_page *mp; - - if (mc->mc_snum < 2) { - return MDB_NOTFOUND; /* root has no siblings */ - } - - mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Z"u, index %u", - mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); - - if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) - : (mc->mc_ki[mc->mc_top] == 0)) { - DPRINTF(("no more keys left, moving to %s sibling", - move_right ? "right" : "left")); - if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { - /* undo cursor_pop before returning */ - mc->mc_top++; - mc->mc_snum++; - return rc; - } - } else { - if (move_right) - mc->mc_ki[mc->mc_top]++; - else - mc->mc_ki[mc->mc_top]--; - DPRINTF(("just moving to %s index key %u", - move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); - } - mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - - indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) { - /* mc will be inconsistent if caller does mc_snum++ as above */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - return rc; - } - - mdb_cursor_push(mc, mp); - if (!move_right) - mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; - - return MDB_SUCCESS; -} - -/** Move the cursor to the next data item. */ -static int -mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - if (mc->mc_flags & C_EOF) { - return MDB_NOTFOUND; - } - - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_NEXT || op == MDB_NEXT_DUP) { - rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); - if (op != MDB_NEXT || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) - MDB_GET_KEY(leaf, key); - return rc; - } - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_NEXT_DUP) - return MDB_NOTFOUND; - } - } - - DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - if (mc->mc_flags & C_DEL) - goto skip; - - if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { - DPUTS("=====> move to next sibling page"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { - mc->mc_flags |= C_EOF; - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]++; - -skip: - DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the previous data item. */ -static int -mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_PREV || op == MDB_PREV_DUP) { - rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); - if (op != MDB_PREV || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) { - MDB_GET_KEY(leaf, key); - mc->mc_flags &= ~C_EOF; - } - return rc; - } - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_PREV_DUP) - return MDB_NOTFOUND; - } - } - - DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - - if (mc->mc_ki[mc->mc_top] == 0) { - DPUTS("=====> move to prev sibling page"); - if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]--; - - mc->mc_flags &= ~C_EOF; - - DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Set the cursor on a specific data item. */ -static int -mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op, int *exactp) -{ - int rc; - MDB_page *mp; - MDB_node *leaf = NULL; - DKBUF; - - if (key->mv_size == 0) - return MDB_BAD_VALSIZE; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - /* See if we're already on the right page */ - if (mc->mc_flags & C_INITIALIZED) { - MDB_val nodekey; - - mp = mc->mc_pg[mc->mc_top]; - if (!NUMKEYS(mp)) { - mc->mc_ki[mc->mc_top] = 0; - return MDB_NOTFOUND; - } - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_size = mc->mc_db->md_pad; - nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, 0); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* Probably happens rarely, but first node on the page - * was the one we wanted. - */ - mc->mc_ki[mc->mc_top] = 0; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc > 0) { - unsigned int i; - unsigned int nkeys = NUMKEYS(mp); - if (nkeys > 1) { - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - nkeys-1, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, nkeys-1); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* last node was the one we wanted */ - mc->mc_ki[mc->mc_top] = nkeys-1; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc < 0) { - if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { - /* This is definitely the right page, skip search_page */ - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - mc->mc_ki[mc->mc_top], nodekey.mv_size); - } else { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* current node was the one we wanted */ - if (exactp) - *exactp = 1; - goto set1; - } - } - rc = 0; - goto set2; - } - } - /* If any parents have right-sibs, search. - * Otherwise, there's nothing further. - */ - for (i=0; imc_top; i++) - if (mc->mc_ki[i] < - NUMKEYS(mc->mc_pg[i])-1) - break; - if (i == mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = nkeys; - return MDB_NOTFOUND; - } - } - if (!mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = 0; - if (op == MDB_SET_RANGE && !exactp) { - rc = 0; - goto set1; - } else - return MDB_NOTFOUND; - } - } - - rc = mdb_page_search(mc, key, 0); - if (rc != MDB_SUCCESS) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - -set2: - leaf = mdb_node_search(mc, key, exactp); - if (exactp != NULL && !*exactp) { - /* MDB_SET specified and not an exact match. */ - return MDB_NOTFOUND; - } - - if (leaf == NULL) { - DPUTS("===> inexact leaf not found, goto sibling"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) - return rc; /* no entries matched */ - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, 0); - } - -set1: - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - if (IS_LEAF2(mp)) { - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } - return MDB_SUCCESS; - } - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - } else { - int ex2, *ex2p; - if (op == MDB_GET_BOTH) { - ex2p = &ex2; - ex2 = 0; - } else { - ex2p = NULL; - } - rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); - if (rc != MDB_SUCCESS) - return rc; - } - } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { - MDB_val olddata; - MDB_cmp_func *dcmp; - if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS) - return rc; - dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) - dcmp = mdb_cmp_clong; -#endif - rc = dcmp(data, &olddata); - if (rc) { - if (op == MDB_GET_BOTH || rc > 0) - return MDB_NOTFOUND; - rc = 0; - *data = olddata; - } - - } else { - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - /* The key already matches in all other cases */ - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) - MDB_GET_KEY(leaf, key); - DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); - - return rc; -} - -/** Move the cursor to the first item in the database. */ -static int -mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - mc->mc_ki[mc->mc_top] = 0; - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the last item in the database. */ -static int -mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - - if (!(mc->mc_flags & C_EOF)) { - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_LAST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - } - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; - mc->mc_flags |= C_INITIALIZED|C_EOF; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -int -mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op) -{ - int rc; - int exact = 0; - int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); - - if (mc == NULL) - return EINVAL; - - if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - switch (op) { - case MDB_GET_CURRENT: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - } else { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int nkeys = NUMKEYS(mp); - if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { - mc->mc_ki[mc->mc_top] = nkeys; - rc = MDB_NOTFOUND; - break; - } - rc = MDB_SUCCESS; - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } else { - MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY(leaf, key); - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (mc->mc_flags & C_DEL) - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); - } else { - rc = mdb_node_read(mc->mc_txn, leaf, data); - } - } - } - } - break; - case MDB_GET_BOTH: - case MDB_GET_BOTH_RANGE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - /* FALLTHRU */ - case MDB_SET: - case MDB_SET_KEY: - case MDB_SET_RANGE: - if (key == NULL) { - rc = EINVAL; - } else { - rc = mdb_cursor_set(mc, key, data, op, - op == MDB_SET_RANGE ? NULL : &exact); - } - break; - case MDB_GET_MULTIPLE: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - rc = MDB_SUCCESS; - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || - (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) - break; - goto fetchm; - case MDB_NEXT_MULTIPLE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); - if (rc == MDB_SUCCESS) { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - MDB_cursor *mx; -fetchm: - mx = &mc->mc_xcursor->mx_cursor; - data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * - mx->mc_db->md_pad; - data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); - mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; - } else { - rc = MDB_NOTFOUND; - } - } - break; - case MDB_NEXT: - case MDB_NEXT_DUP: - case MDB_NEXT_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, op); - break; - case MDB_PREV: - case MDB_PREV_DUP: - case MDB_PREV_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = mdb_cursor_last(mc, key, data); - if (rc) - break; - mc->mc_flags |= C_INITIALIZED; - mc->mc_ki[mc->mc_top]++; - } - rc = mdb_cursor_prev(mc, key, data, op); - break; - case MDB_FIRST: - rc = mdb_cursor_first(mc, key, data); - break; - case MDB_FIRST_DUP: - mfunc = mdb_cursor_first; - mmove: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - { - MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - MDB_GET_KEY(leaf, key); - rc = mdb_node_read(mc->mc_txn, leaf, data); - break; - } - } - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); - break; - case MDB_LAST: - rc = mdb_cursor_last(mc, key, data); - break; - case MDB_LAST_DUP: - mfunc = mdb_cursor_last; - goto mmove; - default: - DPRINTF(("unhandled/unimplemented cursor operation %u", op)); - rc = EINVAL; - break; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - return rc; -} - -/** Touch all the pages in the cursor stack. Set mc_top. - * Makes sure all the pages are writable, before attempting a write operation. - * @param[in] mc The cursor to operate on. - */ -static int -mdb_cursor_touch(MDB_cursor *mc) -{ - int rc = MDB_SUCCESS; - - if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { - MDB_cursor mc2; - MDB_xcursor mcx; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); - if (rc) - return rc; - *mc->mc_dbflag |= DB_DIRTY; - } - mc->mc_top = 0; - if (mc->mc_snum) { - do { - rc = mdb_page_touch(mc); - } while (!rc && ++(mc->mc_top) < mc->mc_snum); - mc->mc_top = mc->mc_snum-1; - } - return rc; -} - -/** Do not spill pages to disk if txn is getting full, may fail instead */ -#define MDB_NOSPILL 0x8000 - -int -mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, - unsigned int flags) -{ - enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ - MDB_env *env; - MDB_node *leaf = NULL; - MDB_page *fp, *mp; - uint16_t fp_flags; - MDB_val xdata, *rdata, dkey, olddata; - MDB_db dummy; - int do_sub = 0, insert_key, insert_data; - unsigned int mcount = 0, dcount = 0, nospill; - size_t nsize; - int rc, rc2; - unsigned int nflags; - DKBUF; - - if (mc == NULL || key == NULL) - return EINVAL; - - env = mc->mc_txn->mt_env; - - /* Check this first so counter will always be zero on any - * early failures. - */ - if (flags & MDB_MULTIPLE) { - dcount = data[1].mv_size; - data[1].mv_size = 0; - if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) - return MDB_INCOMPATIBLE; - } - - nospill = flags & MDB_NOSPILL; - flags &= ~MDB_NOSPILL; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (key->mv_size-1 >= ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; - -#if SIZE_MAX > MAXDATASIZE - if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) - return MDB_BAD_VALSIZE; -#else - if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; -#endif - - DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", - DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); - - dkey.mv_size = 0; - - if (flags == MDB_CURRENT) { - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - rc = MDB_SUCCESS; - } else if (mc->mc_db->md_root == P_INVALID) { - /* new database, cursor has nothing to point to */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - rc = MDB_NO_ROOT; - } else { - int exact = 0; - MDB_val d2; - if (flags & MDB_APPEND) { - MDB_val k2; - rc = mdb_cursor_last(mc, &k2, &d2); - if (rc == 0) { - rc = mc->mc_dbx->md_cmp(key, &k2); - if (rc > 0) { - rc = MDB_NOTFOUND; - mc->mc_ki[mc->mc_top]++; - } else { - /* new key is <= last key */ - rc = MDB_KEYEXIST; - } - } - } else { - rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); - } - if ((flags & MDB_NOOVERWRITE) && rc == 0) { - DPRINTF(("duplicate key [%s]", DKEY(key))); - *data = d2; - return MDB_KEYEXIST; - } - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - /* Cursor is positioned, check for room in the dirty list */ - if (!nospill) { - if (flags & MDB_MULTIPLE) { - rdata = &xdata; - xdata.mv_size = data->mv_size * dcount; - } else { - rdata = data; - } - if ((rc2 = mdb_page_spill(mc, key, rdata))) - return rc2; - } - - if (rc == MDB_NO_ROOT) { - MDB_page *np; - /* new database, write a root leaf page */ - DPUTS("allocating new root leaf page"); - if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { - return rc2; - } - mdb_cursor_push(mc, np); - mc->mc_db->md_root = np->mp_pgno; - mc->mc_db->md_depth++; - *mc->mc_dbflag |= DB_DIRTY; - if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) - == MDB_DUPFIXED) - np->mp_flags |= P_LEAF2; - mc->mc_flags |= C_INITIALIZED; - } else { - /* make sure all cursor pages are writable */ - rc2 = mdb_cursor_touch(mc); - if (rc2) - return rc2; - } - - insert_key = insert_data = rc; - if (insert_key) { - /* The key does not exist */ - DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - LEAFSIZE(key, data) > env->me_nodemax) - { - /* Too big for a node, insert in sub-DB. Set up an empty - * "old sub-page" for prep_subDB to expand to a full page. - */ - fp_flags = P_LEAF|P_DIRTY; - fp = env->me_pbuf; - fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ - fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); - olddata.mv_size = PAGEHDRSZ; - goto prep_subDB; - } - } else { - /* there's only a key anyway, so this is a no-op */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - char *ptr; - unsigned int ksize = mc->mc_db->md_pad; - if (key->mv_size != ksize) - return MDB_BAD_VALSIZE; - ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); - memcpy(ptr, key->mv_data, ksize); -fix_parent: - /* if overwriting slot 0 of leaf, need to - * update branch key if there is a parent page - */ - if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { - unsigned short top = mc->mc_top; - mc->mc_top--; - /* slot 0 is always an empty key, find real slot */ - while (mc->mc_top && !mc->mc_ki[mc->mc_top]) - mc->mc_top--; - if (mc->mc_ki[mc->mc_top]) - rc2 = mdb_update_key(mc, key); - else - rc2 = MDB_SUCCESS; - mc->mc_top = top; - if (rc2) - return rc2; - } - return MDB_SUCCESS; - } - -more: - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - olddata.mv_size = NODEDSZ(leaf); - olddata.mv_data = NODEDATA(leaf); - - /* DB has dups? */ - if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { - /* Prepare (sub-)page/sub-DB to accept the new item, - * if needed. fp: old sub-page or a header faking - * it. mp: new (sub-)page. offset: growth in page - * size. xdata: node data with new page or DB. - */ - unsigned i, offset = 0; - mp = fp = xdata.mv_data = env->me_pbuf; - mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; - - /* Was a single item before, must convert now */ - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - MDB_cmp_func *dcmp; - /* Just overwrite the current item */ - if (flags == MDB_CURRENT) - goto current; - dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) - dcmp = mdb_cmp_clong; -#endif - /* does data match? */ - if (!dcmp(data, &olddata)) { - if (flags & MDB_NODUPDATA) - return MDB_KEYEXIST; - /* overwrite it */ - goto current; - } - - /* Back up original data item */ - dkey.mv_size = olddata.mv_size; - dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); - - /* Make sub-page header for the dup items, with dummy body */ - fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; - fp->mp_lower = (PAGEHDRSZ-PAGEBASE); - xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp->mp_flags |= P_LEAF2; - fp->mp_pad = data->mv_size; - xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ - } else { - xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + - (dkey.mv_size & 1) + (data->mv_size & 1); - } - fp->mp_upper = xdata.mv_size - PAGEBASE; - olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ - } else if (leaf->mn_flags & F_SUBDATA) { - /* Data is on sub-DB, just store it */ - flags |= F_DUPDATA|F_SUBDATA; - goto put_sub; - } else { - /* Data is on sub-page */ - fp = olddata.mv_data; - switch (flags) { - default: - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - offset = EVEN(NODESIZE + sizeof(indx_t) + - data->mv_size); - break; - } - offset = fp->mp_pad; - if (SIZELEFT(fp) < offset) { - offset *= 4; /* space for 4 more */ - break; - } - /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ - case MDB_CURRENT: - fp->mp_flags |= P_DIRTY; - COPY_PGNO(fp->mp_pgno, mp->mp_pgno); - mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; - flags |= F_DUPDATA; - goto put_sub; - } - xdata.mv_size = olddata.mv_size + offset; - } - - fp_flags = fp->mp_flags; - if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { - /* Too big for a sub-page, convert to sub-DB */ - fp_flags &= ~P_SUBP; -prep_subDB: - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp_flags |= P_LEAF2; - dummy.md_pad = fp->mp_pad; - dummy.md_flags = MDB_DUPFIXED; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - dummy.md_flags |= MDB_INTEGERKEY; - } else { - dummy.md_pad = 0; - dummy.md_flags = 0; - } - dummy.md_depth = 1; - dummy.md_branch_pages = 0; - dummy.md_leaf_pages = 1; - dummy.md_overflow_pages = 0; - dummy.md_entries = NUMKEYS(fp); - xdata.mv_size = sizeof(MDB_db); - xdata.mv_data = &dummy; - if ((rc = mdb_page_alloc(mc, 1, &mp))) - return rc; - offset = env->me_psize - olddata.mv_size; - flags |= F_DUPDATA|F_SUBDATA; - dummy.md_root = mp->mp_pgno; - } - if (mp != fp) { - mp->mp_flags = fp_flags | P_DIRTY; - mp->mp_pad = fp->mp_pad; - mp->mp_lower = fp->mp_lower; - mp->mp_upper = fp->mp_upper + offset; - if (fp_flags & P_LEAF2) { - memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); - } else { - memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, - olddata.mv_size - fp->mp_upper - PAGEBASE); - for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; - } - } - - rdata = &xdata; - flags |= F_DUPDATA; - do_sub = 1; - if (!insert_key) - mdb_node_del(mc, 0); - goto new_sub; - } -current: - /* overflow page overwrites need special handling */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); - - memcpy(&pg, olddata.mv_data, sizeof(pg)); - if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) - return rc2; - ovpages = omp->mp_pages; - - /* Is the ov page large enough? */ - if (ovpages >= dpages) { - if (!(omp->mp_flags & P_DIRTY) && - (level || (env->me_flags & MDB_WRITEMAP))) - { - rc = mdb_page_unspill(mc->mc_txn, omp, &omp); - if (rc) - return rc; - level = 0; /* dirty in this txn or clean */ - } - /* Is it dirty? */ - if (omp->mp_flags & P_DIRTY) { - /* yes, overwrite it. Note in this case we don't - * bother to try shrinking the page if the new data - * is smaller than the overflow threshold. - */ - if (level > 1) { - /* It is writable only in a parent txn */ - size_t sz = (size_t) env->me_psize * ovpages, off; - MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); - MDB_ID2 id2; - if (!np) - return ENOMEM; - id2.mid = pg; - id2.mptr = np; - rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); - mdb_cassert(mc, rc2 == 0); - if (!(flags & MDB_RESERVE)) { - /* Copy end of page, adjusting alignment so - * compiler may copy words instead of bytes. - */ - off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); - memcpy((size_t *)((char *)np + off), - (size_t *)((char *)omp + off), sz - off); - sz = PAGEHDRSZ; - } - memcpy(np, omp, sz); /* Copy beginning of page */ - omp = np; - } - SETDSZ(leaf, data->mv_size); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = METADATA(omp); - else - memcpy(METADATA(omp), data->mv_data, data->mv_size); - return MDB_SUCCESS; - } - } - if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) - return rc2; - } else if (data->mv_size == olddata.mv_size) { - /* same size, just replace it. Note that we could - * also reuse this node if the new data is smaller, - * but instead we opt to shrink the node in that case. - */ - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = olddata.mv_data; - else if (!(mc->mc_flags & C_SUB)) - memcpy(olddata.mv_data, data->mv_data, data->mv_size); - else { - memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); - goto fix_parent; - } - return MDB_SUCCESS; - } - mdb_node_del(mc, 0); - } - - rdata = data; - -new_sub: - nflags = flags & NODE_ADD_FLAGS; - nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); - if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { - if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) - nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ - if (!insert_key) - nflags |= MDB_SPLIT_REPLACE; - rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); - } else { - /* There is room already in this leaf page. */ - rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); - if (rc == 0 && insert_key) { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { - m3->mc_ki[i]++; - } - } - } - } - - if (rc == MDB_SUCCESS) { - /* Now store the actual data in the child DB. Note that we're - * storing the user data in the keys field, so there are strict - * size limits on dupdata. The actual data fields of the child - * DB are all zero size. - */ - if (do_sub) { - int xflags; - size_t ecount; -put_sub: - xdata.mv_size = 0; - xdata.mv_data = ""; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags & MDB_CURRENT) { - xflags = MDB_CURRENT|MDB_NOSPILL; - } else { - mdb_xcursor_init1(mc, leaf); - xflags = (flags & MDB_NODUPDATA) ? - MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; - } - /* converted, write the original data first */ - if (dkey.mv_size) { - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); - if (rc) - goto bad_sub; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init1(m2, leaf); - } - } - } - /* we've done our job */ - dkey.mv_size = 0; - } - ecount = mc->mc_xcursor->mx_db.md_entries; - if (flags & MDB_APPENDDUP) - xflags |= MDB_APPEND; - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); - if (flags & F_SUBDATA) { - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } - insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; - } - /* Increment count unless we just replaced an existing item. */ - if (insert_data) - mc->mc_db->md_entries++; - if (insert_key) { - /* Invalidate txn if we created an empty sub-DB */ - if (rc) - goto bad_sub; - /* If we succeeded and the key didn't exist before, - * make sure the cursor is marked valid. - */ - mc->mc_flags |= C_INITIALIZED; - } - if (flags & MDB_MULTIPLE) { - if (!rc) { - mcount++; - /* let caller know how many succeeded, if any */ - data[1].mv_size = mcount; - if (mcount < dcount) { - data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; - insert_key = insert_data = 0; - goto more; - } - } - } - return rc; -bad_sub: - if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ - rc = MDB_CORRUPTED; - } - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_cursor_del(MDB_cursor *mc, unsigned int flags) -{ - MDB_node *leaf; - MDB_page *mp; - int rc; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) - return MDB_NOTFOUND; - - if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) - return rc; - - rc = mdb_cursor_touch(mc); - if (rc) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - if (IS_LEAF2(mp)) - goto del_key; - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (flags & MDB_NODUPDATA) { - /* mdb_cursor_del0() will subtract the final entry */ - mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; - } else { - if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); - if (rc) - return rc; - /* If sub-DB still has entries, we're done */ - if (mc->mc_xcursor->mx_db.md_entries) { - if (leaf->mn_flags & F_SUBDATA) { - /* update subDB info */ - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } else { - MDB_cursor *m2; - /* shrink fake page */ - mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - /* fix other sub-DB cursors pointed at this fake page */ - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - } - mc->mc_db->md_entries--; - mc->mc_flags |= C_DEL; - return rc; - } - /* otherwise fall thru and delete the sub-DB */ - } - - if (leaf->mn_flags & F_SUBDATA) { - /* add all the child DB's pages to the free list */ - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto fail; - } - } - - /* add overflow pages to free list */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - - memcpy(&pg, NODEDATA(leaf), sizeof(pg)); - if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || - (rc = mdb_ovpage_free(mc, omp))) - goto fail; - } - -del_key: - return mdb_cursor_del0(mc); - -fail: - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Allocate and initialize new pages for a database. - * @param[in] mc a cursor on the database being added to. - * @param[in] flags flags defining what type of page is being allocated. - * @param[in] num the number of pages to allocate. This is usually 1, - * unless allocating overflow pages for a large record. - * @param[out] mp Address of a page, or NULL on failure. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) -{ - MDB_page *np; - int rc; - - if ((rc = mdb_page_alloc(mc, num, &np))) - return rc; - DPRINTF(("allocated new mpage %"Z"u, page size %u", - np->mp_pgno, mc->mc_txn->mt_env->me_psize)); - np->mp_flags = flags | P_DIRTY; - np->mp_lower = (PAGEHDRSZ-PAGEBASE); - np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; - - if (IS_BRANCH(np)) - mc->mc_db->md_branch_pages++; - else if (IS_LEAF(np)) - mc->mc_db->md_leaf_pages++; - else if (IS_OVERFLOW(np)) { - mc->mc_db->md_overflow_pages += num; - np->mp_pages = num; - } - *mp = np; - - return 0; -} - -/** Calculate the size of a leaf node. - * The size depends on the environment's page size; if a data item - * is too large it will be put onto an overflow page and the node - * size will only include the key and not the data. Sizes are always - * rounded up to an even number of bytes, to guarantee 2-byte alignment - * of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @param[in] data The data for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) -{ - size_t sz; - - sz = LEAFSIZE(key, data); - if (sz > env->me_nodemax) { - /* put on overflow page */ - sz -= data->mv_size - sizeof(pgno_t); - } - - return EVEN(sz + sizeof(indx_t)); -} - -/** Calculate the size of a branch node. - * The size should depend on the environment's page size but since - * we currently don't support spilling large keys onto overflow - * pages, it's simply the size of the #MDB_node header plus the - * size of the key. Sizes are always rounded up to an even number - * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_branch_size(MDB_env *env, MDB_val *key) -{ - size_t sz; - - sz = INDXSIZE(key); - if (sz > env->me_nodemax) { - /* put on overflow page */ - /* not implemented */ - /* sz -= key->size - sizeof(pgno_t); */ - } - - return sz + sizeof(indx_t); -} - -/** Add a node to the page pointed to by the cursor. - * @param[in] mc The cursor for this operation. - * @param[in] indx The index on the page where the new node should be added. - * @param[in] key The key for the new node. - * @param[in] data The data for the new node, if any. - * @param[in] pgno The page number, if adding a branch node. - * @param[in] flags Flags for the node. - * @return 0 on success, non-zero on failure. Possible errors are: - *

ENOMEM - failed to allocate overflow pages for the node. - *
MDB_PAGE_FULL - there is insufficient room in the page. This error - * should never happen since all callers already calculate the - * page's free space before calling this function. - *

- */ -static int -mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) -{ - unsigned int i; - size_t node_size = NODESIZE; - ssize_t room; - indx_t ofs; - MDB_node *node; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_page *ofp = NULL; /* overflow page */ - DKBUF; - - mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - - DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", - IS_LEAF(mp) ? "leaf" : "branch", - IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, - key ? key->mv_size : 0, key ? DKEY(key) : "null")); - - if (IS_LEAF2(mp)) { - /* Move higher keys up one slot. */ - int ksize = mc->mc_db->md_pad, dif; - char *ptr = LEAF2KEY(mp, indx, ksize); - dif = NUMKEYS(mp) - indx; - if (dif > 0) - memmove(ptr+ksize, ptr, dif*ksize); - /* insert new key */ - memcpy(ptr, key->mv_data, ksize); - - /* Just using these for counting */ - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - return MDB_SUCCESS; - } - - room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); - if (key != NULL) - node_size += key->mv_size; - if (IS_LEAF(mp)) { - mdb_cassert(mc, data); - if (F_ISSET(flags, F_BIGDATA)) { - /* Data already on overflow page. */ - node_size += sizeof(pgno_t); - } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { - int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); - int rc; - /* Put data on overflow page. */ - DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", - data->mv_size, node_size+data->mv_size)); - node_size = EVEN(node_size + sizeof(pgno_t)); - if ((ssize_t)node_size > room) - goto full; - if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) - return rc; - DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); - flags |= F_BIGDATA; - goto update; - } else { - node_size += data->mv_size; - } - } - node_size = EVEN(node_size); - if ((ssize_t)node_size > room) - goto full; - -update: - /* Move higher pointers up one slot. */ - for (i = NUMKEYS(mp); i > indx; i--) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; - - /* Adjust free space offsets. */ - ofs = mp->mp_upper - node_size; - mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); - mp->mp_ptrs[indx] = ofs; - mp->mp_upper = ofs; - mp->mp_lower += sizeof(indx_t); - - /* Write the node data. */ - node = NODEPTR(mp, indx); - node->mn_ksize = (key == NULL) ? 0 : key->mv_size; - node->mn_flags = flags; - if (IS_LEAF(mp)) - SETDSZ(node,data->mv_size); - else - SETPGNO(node,pgno); - - if (key) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - if (IS_LEAF(mp)) { - mdb_cassert(mc, key); - if (ofp == NULL) { - if (F_ISSET(flags, F_BIGDATA)) - memcpy(node->mn_data + key->mv_size, data->mv_data, - sizeof(pgno_t)); - else if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = node->mn_data + key->mv_size; - else - memcpy(node->mn_data + key->mv_size, data->mv_data, - data->mv_size); - } else { - memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, - sizeof(pgno_t)); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = METADATA(ofp); - else - memcpy(METADATA(ofp), data->mv_data, data->mv_size); - } - } - - return MDB_SUCCESS; - -full: - DPRINTF(("not enough room in page %"Z"u, got %u ptrs", - mdb_dbg_pgno(mp), NUMKEYS(mp))); - DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); - DPRINTF(("node size = %"Z"u", node_size)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_FULL; -} - -/** Delete the specified node from a page. - * @param[in] mc Cursor pointing to the node to delete. - * @param[in] ksize The size of a node. Only used if the page is - * part of a #MDB_DUPFIXED database. - */ -static void -mdb_node_del(MDB_cursor *mc, int ksize) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - indx_t indx = mc->mc_ki[mc->mc_top]; - unsigned int sz; - indx_t i, j, numkeys, ptr; - MDB_node *node; - char *base; - - DPRINTF(("delete node %u on %s page %"Z"u", indx, - IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); - numkeys = NUMKEYS(mp); - mdb_cassert(mc, indx < numkeys); - - if (IS_LEAF2(mp)) { - int x = numkeys - 1 - indx; - base = LEAF2KEY(mp, indx, ksize); - if (x) - memmove(base, base + ksize, x * ksize); - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += ksize - sizeof(indx_t); - return; - } - - node = NODEPTR(mp, indx); - sz = NODESIZE + node->mn_ksize; - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - sz += sizeof(pgno_t); - else - sz += NODEDSZ(node); - } - sz = EVEN(sz); - - ptr = mp->mp_ptrs[indx]; - for (i = j = 0; i < numkeys; i++) { - if (i != indx) { - mp->mp_ptrs[j] = mp->mp_ptrs[i]; - if (mp->mp_ptrs[i] < ptr) - mp->mp_ptrs[j] += sz; - j++; - } - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + sz, base, ptr - mp->mp_upper); - - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += sz; -} - -/** Compact the main page after deleting a node on a subpage. - * @param[in] mp The main page to operate on. - * @param[in] indx The index of the subpage on the main page. - */ -static void -mdb_node_shrink(MDB_page *mp, indx_t indx) -{ - MDB_node *node; - MDB_page *sp, *xp; - char *base; - int nsize, delta; - indx_t i, numkeys, ptr; - - node = NODEPTR(mp, indx); - sp = (MDB_page *)NODEDATA(node); - delta = SIZELEFT(sp); - xp = (MDB_page *)((char *)sp + delta); - - /* shift subpage upward */ - if (IS_LEAF2(sp)) { - nsize = NUMKEYS(sp) * sp->mp_pad; - if (nsize & 1) - return; /* do not make the node uneven-sized */ - memmove(METADATA(xp), METADATA(sp), nsize); - } else { - int i; - numkeys = NUMKEYS(sp); - for (i=numkeys-1; i>=0; i--) - xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; - } - xp->mp_upper = sp->mp_lower; - xp->mp_lower = sp->mp_lower; - xp->mp_flags = sp->mp_flags; - xp->mp_pad = sp->mp_pad; - COPY_PGNO(xp->mp_pgno, mp->mp_pgno); - - nsize = NODEDSZ(node) - delta; - SETDSZ(node, nsize); - - /* shift lower nodes upward */ - ptr = mp->mp_ptrs[indx]; - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] += delta; - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); - mp->mp_upper += delta; -} - -/** Initial setup of a sorted-dups cursor. - * Sorted duplicates are implemented as a sub-database for the given key. - * The duplicate data items are actually keys of the sub-database. - * Operations on the duplicate data items are performed using a sub-cursor - * initialized when the sub-database is first accessed. This function does - * the preliminary setup of the sub-cursor, filling in the fields that - * depend only on the parent DB. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - */ -static void -mdb_xcursor_init0(MDB_cursor *mc) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - mx->mx_cursor.mc_xcursor = NULL; - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_db = &mx->mx_db; - mx->mx_cursor.mc_dbx = &mx->mx_dbx; - mx->mx_cursor.mc_dbi = mc->mc_dbi; - mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; - mx->mx_dbx.md_name.mv_size = 0; - mx->mx_dbx.md_name.mv_data = NULL; - mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; - mx->mx_dbx.md_dcmp = NULL; - mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; -} - -/** Final setup of a sorted-dups cursor. - * Sets up the fields that depend on the data from the main cursor. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - * @param[in] node The data containing the #MDB_db record for the - * sorted-dup database. - */ -static void -mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - if (node->mn_flags & F_SUBDATA) { - memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); - mx->mx_cursor.mc_pg[0] = 0; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; - } else { - MDB_page *fp = NODEDATA(node); - mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; - mx->mx_db.md_flags = 0; - mx->mx_db.md_depth = 1; - mx->mx_db.md_branch_pages = 0; - mx->mx_db.md_leaf_pages = 1; - mx->mx_db.md_overflow_pages = 0; - mx->mx_db.md_entries = NUMKEYS(fp); - COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; - mx->mx_cursor.mc_pg[0] = fp; - mx->mx_cursor.mc_ki[0] = 0; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - mx->mx_db.md_flags = MDB_DUPFIXED; - mx->mx_db.md_pad = fp->mp_pad; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - mx->mx_db.md_flags |= MDB_INTEGERKEY; - } - } - DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, - mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX - if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) - mx->mx_dbx.md_cmp = mdb_cmp_clong; -#endif -} - -/** Initialize a cursor for a given transaction and database. */ -static void -mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) -{ - mc->mc_next = NULL; - mc->mc_backup = NULL; - mc->mc_dbi = dbi; - mc->mc_txn = txn; - mc->mc_db = &txn->mt_dbs[dbi]; - mc->mc_dbx = &txn->mt_dbxs[dbi]; - mc->mc_dbflag = &txn->mt_dbflags[dbi]; - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_pg[0] = 0; - mc->mc_ki[0] = 0; - mc->mc_flags = 0; - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { - mdb_tassert(txn, mx != NULL); - mc->mc_xcursor = mx; - mdb_xcursor_init0(mc); - } else { - mc->mc_xcursor = NULL; - } - if (*mc->mc_dbflag & DB_STALE) { - mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); - } -} - -int -mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) -{ - MDB_cursor *mc; - size_t size = sizeof(MDB_cursor); - - if (!ret || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - /* Allow read access to the freelist */ - if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EINVAL; - - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) - size += sizeof(MDB_xcursor); - - if ((mc = malloc(size)) != NULL) { - mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); - if (txn->mt_cursors) { - mc->mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = mc; - mc->mc_flags |= C_UNTRACK; - } - } else { - return ENOMEM; - } - - *ret = mc; - - return MDB_SUCCESS; -} - -int -mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) -{ - if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) - return EINVAL; - - if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); - return MDB_SUCCESS; -} - -/* Return the count of duplicate data items for the current key */ -int -mdb_cursor_count(MDB_cursor *mc, size_t *countp) -{ - MDB_node *leaf; - - if (mc == NULL || countp == NULL) - return EINVAL; - - if (mc->mc_xcursor == NULL) - return MDB_INCOMPATIBLE; - - if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (!mc->mc_snum || (mc->mc_flags & C_EOF)) - return MDB_NOTFOUND; - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - *countp = 1; - } else { - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - return EINVAL; - - *countp = mc->mc_xcursor->mx_db.md_entries; - } - return MDB_SUCCESS; -} - -void -mdb_cursor_close(MDB_cursor *mc) -{ - if (mc && !mc->mc_backup) { - /* remove from txn, if tracked */ - if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { - MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) prev = &(*prev)->mc_next; - if (*prev == mc) - *prev = mc->mc_next; - } - free(mc); - } -} - -MDB_txn * -mdb_cursor_txn(MDB_cursor *mc) -{ - if (!mc) return NULL; - return mc->mc_txn; -} - -MDB_dbi -mdb_cursor_dbi(MDB_cursor *mc) -{ - return mc->mc_dbi; -} - -/** Replace the key for a branch node with a new key. - * @param[in] mc Cursor pointing to the node to operate on. - * @param[in] key The new key to use. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_update_key(MDB_cursor *mc, MDB_val *key) -{ - MDB_page *mp; - MDB_node *node; - char *base; - size_t len; - int delta, ksize, oksize; - indx_t ptr, i, numkeys, indx; - DKBUF; - - indx = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - node = NODEPTR(mp, indx); - ptr = mp->mp_ptrs[indx]; -#if MDB_DEBUG - { - MDB_val k2; - char kbuf2[DKBUF_MAXKEYSIZE*2+1]; - k2.mv_data = NODEKEY(node); - k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", - indx, ptr, - mdb_dkey(&k2, kbuf2), - DKEY(key), - mp->mp_pgno)); - } -#endif - - /* Sizes must be 2-byte aligned. */ - ksize = EVEN(key->mv_size); - oksize = EVEN(node->mn_ksize); - delta = ksize - oksize; - - /* Shift node contents if EVEN(key length) changed. */ - if (delta) { - if (delta > 0 && SIZELEFT(mp) < delta) { - pgno_t pgno; - /* not enough space left, do a delete and split */ - DPRINTF(("Not enough room, delta = %d, splitting...", delta)); - pgno = NODEPGNO(node); - mdb_node_del(mc, 0); - return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); - } - - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] -= delta; - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - len = ptr - mp->mp_upper + NODESIZE; - memmove(base - delta, base, len); - mp->mp_upper -= delta; - - node = NODEPTR(mp, indx); - } - - /* But even if no shift was needed, update ksize */ - if (node->mn_ksize != key->mv_size) - node->mn_ksize = key->mv_size; - - if (key->mv_size) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - return MDB_SUCCESS; -} - -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); - -/** Move a node from csrc to cdst. - */ -static int -mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) -{ - MDB_node *srcnode; - MDB_val key, data; - pgno_t srcpg; - MDB_cursor mn; - int rc; - unsigned short flags; - - DKBUF; - - /* Mark src and dst as dirty. */ - if ((rc = mdb_page_touch(csrc)) || - (rc = mdb_page_touch(cdst))) - return rc; - - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); - data.mv_size = 0; - data.mv_data = NULL; - srcpg = 0; - flags = 0; - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); - mdb_cassert(csrc, !((size_t)srcnode & 1)); - srcpg = NODEPGNO(srcnode); - flags = srcnode->mn_flags; - if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - unsigned int snum = csrc->mc_snum; - MDB_node *s2; - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(csrc); - if (rc) - return rc; - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - csrc->mc_snum = snum--; - csrc->mc_top = snum; - } else { - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - } - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { - unsigned int snum = cdst->mc_snum; - MDB_node *s2; - MDB_val bkey; - /* must find the lowest key below dst */ - mdb_cursor_copy(cdst, &mn); - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - bkey.mv_size = mn.mc_db->md_pad; - bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - bkey.mv_size = NODEKSZ(s2); - bkey.mv_data = NODEKEY(s2); - } - mn.mc_snum = snum--; - mn.mc_top = snum; - mn.mc_ki[snum] = 0; - rc = mdb_update_key(&mn, &bkey); - if (rc) - return rc; - } - - DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", - IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", - csrc->mc_ki[csrc->mc_top], - DKEY(&key), - csrc->mc_pg[csrc->mc_top]->mp_pgno, - cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); - - /* Add the node to the destination page. - */ - rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); - if (rc != MDB_SUCCESS) - return rc; - - /* Delete the node from the source page. - */ - mdb_node_del(csrc, key.mv_size); - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mp = csrc->mc_pg[csrc->mc_top]; - - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == - csrc->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - } - } - } - - /* Update the parent separators. - */ - if (csrc->mc_ki[csrc->mc_top] == 0) { - if (csrc->mc_ki[csrc->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for source page %"Z"u to [%s]", - csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(csrc, &mn); - mn.mc_snum--; - mn.mc_top--; - if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) - return rc; - } - if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - MDB_val nullkey; - indx_t ix = csrc->mc_ki[csrc->mc_top]; - nullkey.mv_size = 0; - csrc->mc_ki[csrc->mc_top] = 0; - rc = mdb_update_key(csrc, &nullkey); - csrc->mc_ki[csrc->mc_top] = ix; - mdb_cassert(csrc, rc == MDB_SUCCESS); - } - } - - if (cdst->mc_ki[cdst->mc_top] == 0) { - if (cdst->mc_ki[cdst->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for destination page %"Z"u to [%s]", - cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(cdst, &mn); - mn.mc_snum--; - mn.mc_top--; - if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) - return rc; - } - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { - MDB_val nullkey; - indx_t ix = cdst->mc_ki[cdst->mc_top]; - nullkey.mv_size = 0; - cdst->mc_ki[cdst->mc_top] = 0; - rc = mdb_update_key(cdst, &nullkey); - cdst->mc_ki[cdst->mc_top] = ix; - mdb_cassert(cdst, rc == MDB_SUCCESS); - } - } - - return MDB_SUCCESS; -} - -/** Merge one page into another. - * The nodes from the page pointed to by \b csrc will - * be copied to the page pointed to by \b cdst and then - * the \b csrc page will be freed. - * @param[in] csrc Cursor pointing to the source page. - * @param[in] cdst Cursor pointing to the destination page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) -{ - MDB_page *psrc, *pdst; - MDB_node *srcnode; - MDB_val key, data; - unsigned nkeys; - int rc; - indx_t i, j; - - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - - DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); - - mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ - mdb_cassert(csrc, cdst->mc_snum > 1); - - /* Mark dst as dirty. */ - if ((rc = mdb_page_touch(cdst))) - return rc; - - /* Move all nodes from src to dst. - */ - j = nkeys = NUMKEYS(pdst); - if (IS_LEAF2(psrc)) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = METADATA(psrc); - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); - if (rc != MDB_SUCCESS) - return rc; - key.mv_data = (char *)key.mv_data + key.mv_size; - } - } else { - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - srcnode = NODEPTR(psrc, i); - if (i == 0 && IS_BRANCH(psrc)) { - MDB_cursor mn; - MDB_node *s2; - mdb_cursor_copy(csrc, &mn); - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - key.mv_size = mn.mc_db->md_pad; - key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - } else { - key.mv_size = srcnode->mn_ksize; - key.mv_data = NODEKEY(srcnode); - } - - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); - if (rc != MDB_SUCCESS) - return rc; - } - } - - DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", - pdst->mp_pgno, NUMKEYS(pdst), - (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); - - /* Unlink the src page from parent and add to free list. - */ - csrc->mc_top--; - mdb_node_del(csrc, 0); - if (csrc->mc_ki[csrc->mc_top] == 0) { - key.mv_size = 0; - rc = mdb_update_key(csrc, &key); - if (rc) { - csrc->mc_top++; - return rc; - } - } - csrc->mc_top++; - - psrc = csrc->mc_pg[csrc->mc_top]; - /* If not operating on FreeDB, allow this page to be reused - * in this txn. Otherwise just add to free list. - */ - rc = mdb_page_loose(csrc, psrc); - if (rc) - return rc; - if (IS_LEAF(psrc)) - csrc->mc_db->md_leaf_pages--; - else - csrc->mc_db->md_branch_pages--; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (m3->mc_snum < csrc->mc_snum) continue; - if (m3->mc_pg[csrc->mc_top] == psrc) { - m3->mc_pg[csrc->mc_top] = pdst; - m3->mc_ki[csrc->mc_top] += nkeys; - } - } - } - { - unsigned int snum = cdst->mc_snum; - uint16_t depth = cdst->mc_db->md_depth; - mdb_cursor_pop(cdst); - rc = mdb_rebalance(cdst); - /* Did the tree shrink? */ - if (depth > cdst->mc_db->md_depth) - snum--; - cdst->mc_snum = snum; - cdst->mc_top = snum-1; - } - return rc; -} - -/** Copy the contents of a cursor. - * @param[in] csrc The cursor to copy from. - * @param[out] cdst The cursor to copy to. - */ -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) -{ - unsigned int i; - - cdst->mc_txn = csrc->mc_txn; - cdst->mc_dbi = csrc->mc_dbi; - cdst->mc_db = csrc->mc_db; - cdst->mc_dbx = csrc->mc_dbx; - cdst->mc_snum = csrc->mc_snum; - cdst->mc_top = csrc->mc_top; - cdst->mc_flags = csrc->mc_flags; - - for (i=0; imc_snum; i++) { - cdst->mc_pg[i] = csrc->mc_pg[i]; - cdst->mc_ki[i] = csrc->mc_ki[i]; - } -} - -/** Rebalance the tree after a delete operation. - * @param[in] mc Cursor pointing to the page where rebalancing - * should begin. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_rebalance(MDB_cursor *mc) -{ - MDB_node *node; - int rc; - unsigned int ptop, minkeys; - MDB_cursor mn; - indx_t oldki; - - minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); - DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", - IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); - - if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && - NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); - return MDB_SUCCESS; - } - - if (mc->mc_snum < 2) { - MDB_page *mp = mc->mc_pg[0]; - if (IS_SUBP(mp)) { - DPUTS("Can't rebalance a subpage, ignoring"); - return MDB_SUCCESS; - } - if (NUMKEYS(mp) == 0) { - DPUTS("tree is completely empty"); - mc->mc_db->md_root = P_INVALID; - mc->mc_db->md_depth = 0; - mc->mc_db->md_leaf_pages = 0; - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - /* Adjust cursors pointing to mp */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[0] == mp) { - m3->mc_snum = 0; - m3->mc_top = 0; - m3->mc_flags &= ~C_INITIALIZED; - } - } - } - } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { - int i; - DPUTS("collapsing root page!"); - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); - rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); - if (rc) - return rc; - mc->mc_db->md_depth--; - mc->mc_db->md_branch_pages--; - mc->mc_ki[0] = mc->mc_ki[1]; - for (i = 1; imc_db->md_depth; i++) { - mc->mc_pg[i] = mc->mc_pg[i+1]; - mc->mc_ki[i] = mc->mc_ki[i+1]; - } - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[0] == mp) { - for (i=0; imc_snum; i++) { - m3->mc_pg[i] = m3->mc_pg[i+1]; - m3->mc_ki[i] = m3->mc_ki[i+1]; - } - m3->mc_snum--; - m3->mc_top--; - } - } - } - } else - DPUTS("root page doesn't need rebalancing"); - return MDB_SUCCESS; - } - - /* The parent (branch page) must have at least 2 pointers, - * otherwise the tree is invalid. - */ - ptop = mc->mc_top-1; - mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); - - /* Leaf page fill factor is below the threshold. - * Try to move keys from left or right neighbor, or - * merge with a neighbor page. - */ - - /* Find neighbors. - */ - mdb_cursor_copy(mc, &mn); - mn.mc_xcursor = NULL; - - oldki = mc->mc_ki[mc->mc_top]; - if (mc->mc_ki[ptop] == 0) { - /* We're the leftmost leaf in our parent. - */ - DPUTS("reading right neighbor"); - mn.mc_ki[ptop]++; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = 0; - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); - } else { - /* There is at least one neighbor to the left. - */ - DPUTS("reading left neighbor"); - mn.mc_ki[ptop]--; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; - mc->mc_ki[mc->mc_top] = 0; - } - - DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", - mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); - - /* If the neighbor page is above threshold and has enough keys, - * move one key from it. Otherwise we should try to merge them. - * (A branch page must never have less than 2 keys.) - */ - minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); - if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { - rc = mdb_node_move(&mn, mc); - if (mc->mc_ki[ptop]) { - oldki++; - } - } else { - if (mc->mc_ki[ptop] == 0) { - rc = mdb_page_merge(&mn, mc); - } else { - MDB_cursor dummy; - oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); - mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; - /* We want mdb_rebalance to find mn when doing fixups */ - if (mc->mc_flags & C_SUB) { - dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; - mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy; - dummy.mc_xcursor = (MDB_xcursor *)&mn; - } else { - mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; - mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn; - } - rc = mdb_page_merge(mc, &mn); - if (mc->mc_flags & C_SUB) - mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next; - else - mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next; - mdb_cursor_copy(&mn, mc); - } - mc->mc_flags &= ~C_EOF; - } - mc->mc_ki[mc->mc_top] = oldki; - return rc; -} - -/** Complete a delete operation started by #mdb_cursor_del(). */ -static int -mdb_cursor_del0(MDB_cursor *mc) -{ - int rc; - MDB_page *mp; - indx_t ki; - unsigned int nkeys; - - ki = mc->mc_ki[mc->mc_top]; - mdb_node_del(mc, mc->mc_db->md_pad); - mc->mc_db->md_entries--; - rc = mdb_rebalance(mc); - - if (rc == MDB_SUCCESS) { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - /* DB is totally empty now, just bail out. - * Other cursors adjustments were already done - * by mdb_rebalance and aren't needed here. - */ - if (!mc->mc_snum) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - nkeys = NUMKEYS(mp); - - /* if mc points past last node in page, find next sibling */ - if (mc->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(mc, 1); - if (rc == MDB_NOTFOUND) { - mc->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - - /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { - m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3 == mc || m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= ki) { - m3->mc_flags |= C_DEL; - if (m3->mc_ki[mc->mc_top] > ki) - m3->mc_ki[mc->mc_top]--; - else if (mc->mc_db->md_flags & MDB_DUPSORT) - m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; - } - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(m3, 1); - if (rc == MDB_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - } - } - mc->mc_flags |= C_DEL; - } - - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_del(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) - return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { - /* must ignore any data */ - data = NULL; - } - - return mdb_del0(txn, dbi, key, data, 0); -} - -static int -mdb_del0(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - MDB_cursor_op op; - MDB_val rdata, *xdata; - int rc, exact = 0; - DKBUF; - - DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); - - mdb_cursor_init(&mc, txn, dbi, &mx); - - if (data) { - op = MDB_GET_BOTH; - rdata = *data; - xdata = &rdata; - } else { - op = MDB_SET; - xdata = NULL; - flags |= MDB_NODUPDATA; - } - rc = mdb_cursor_set(&mc, key, xdata, op, &exact); - if (rc == 0) { - /* let mdb_page_split know about this cursor if needed: - * delete will trigger a rebalance; if it needs to move - * a node from one page to another, it will have to - * update the parent's separator key(s). If the new sepkey - * is larger than the current one, the parent page may - * run out of space, triggering a split. We need this - * cursor to be consistent until the end of the rebalance. - */ - mc.mc_flags |= C_UNTRACK; - mc.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &mc; - rc = mdb_cursor_del(&mc, flags); - txn->mt_cursors[dbi] = mc.mc_next; - } - return rc; -} - -/** Split a page and insert a new node. - * @param[in,out] mc Cursor pointing to the page and desired insertion index. - * The cursor will be updated to point to the actual page and index where - * the node got inserted after the split. - * @param[in] newkey The key for the newly inserted node. - * @param[in] newdata The data for the newly inserted node. - * @param[in] newpgno The page number, if the new node is a branch node. - * @param[in] nflags The #NODE_ADD_FLAGS for the new node. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, - unsigned int nflags) -{ - unsigned int flags; - int rc = MDB_SUCCESS, new_root = 0, did_split = 0; - indx_t newindx; - pgno_t pgno = 0; - int i, j, split_indx, nkeys, pmax; - MDB_env *env = mc->mc_txn->mt_env; - MDB_node *node; - MDB_val sepkey, rkey, xdata, *rdata = &xdata; - MDB_page *copy = NULL; - MDB_page *mp, *rp, *pp; - int ptop; - MDB_cursor mn; - DKBUF; - - mp = mc->mc_pg[mc->mc_top]; - newindx = mc->mc_ki[mc->mc_top]; - nkeys = NUMKEYS(mp); - - DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", - IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, - DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); - - /* Create a right sibling. */ - if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) - return rc; - DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); - - if (mc->mc_snum < 2) { - if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) - goto done; - /* shift current top to make room for new parent */ - mc->mc_pg[1] = mc->mc_pg[0]; - mc->mc_ki[1] = mc->mc_ki[0]; - mc->mc_pg[0] = pp; - mc->mc_ki[0] = 0; - mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); - mc->mc_db->md_depth++; - new_root = 1; - - /* Add left (implicit) pointer. */ - if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { - /* undo the pre-push */ - mc->mc_pg[0] = mc->mc_pg[1]; - mc->mc_ki[0] = mc->mc_ki[1]; - mc->mc_db->md_root = mp->mp_pgno; - mc->mc_db->md_depth--; - goto done; - } - mc->mc_snum = 2; - mc->mc_top = 1; - ptop = 0; - } else { - ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); - } - - mc->mc_flags |= C_SPLITTING; - mdb_cursor_copy(mc, &mn); - mn.mc_pg[mn.mc_top] = rp; - mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; - - if (nflags & MDB_APPEND) { - mn.mc_ki[mn.mc_top] = 0; - sepkey = *newkey; - split_indx = newindx; - nkeys = 0; - } else { - - split_indx = (nkeys+1) / 2; - - if (IS_LEAF2(rp)) { - char *split, *ins; - int x; - unsigned int lsize, rsize, ksize; - /* Move half of the keys to the right sibling */ - x = mc->mc_ki[mc->mc_top] - split_indx; - ksize = mc->mc_db->md_pad; - split = LEAF2KEY(mp, split_indx, ksize); - rsize = (nkeys - split_indx) * ksize; - lsize = (nkeys - split_indx) * sizeof(indx_t); - mp->mp_lower -= lsize; - rp->mp_lower += lsize; - mp->mp_upper += rsize - lsize; - rp->mp_upper -= rsize - lsize; - sepkey.mv_size = ksize; - if (newindx == split_indx) { - sepkey.mv_data = newkey->mv_data; - } else { - sepkey.mv_data = split; - } - if (x<0) { - ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); - memcpy(rp->mp_ptrs, split, rsize); - sepkey.mv_data = rp->mp_ptrs; - memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); - memcpy(ins, newkey->mv_data, ksize); - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - } else { - if (x) - memcpy(rp->mp_ptrs, split, x * ksize); - ins = LEAF2KEY(rp, x, ksize); - memcpy(ins, newkey->mv_data, ksize); - memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); - rp->mp_lower += sizeof(indx_t); - rp->mp_upper -= ksize - sizeof(indx_t); - mc->mc_ki[mc->mc_top] = x; - mc->mc_pg[mc->mc_top] = rp; - } - } else { - int psize, nsize, k; - /* Maximum free space in an empty page */ - pmax = env->me_psize - PAGEHDRSZ; - if (IS_LEAF(mp)) - nsize = mdb_leaf_size(env, newkey, newdata); - else - nsize = mdb_branch_size(env, newkey); - nsize = EVEN(nsize); - - /* grab a page to hold a temporary copy */ - copy = mdb_page_malloc(mc->mc_txn, 1); - if (copy == NULL) { - rc = ENOMEM; - goto done; - } - copy->mp_pgno = mp->mp_pgno; - copy->mp_flags = mp->mp_flags; - copy->mp_lower = (PAGEHDRSZ-PAGEBASE); - copy->mp_upper = env->me_psize - PAGEBASE; - - /* prepare to insert */ - for (i=0, j=0; imp_ptrs[j++] = 0; - } - copy->mp_ptrs[j++] = mp->mp_ptrs[i]; - } - - /* When items are relatively large the split point needs - * to be checked, because being off-by-one will make the - * difference between success or failure in mdb_node_add. - * - * It's also relevant if a page happens to be laid out - * such that one half of its nodes are all "small" and - * the other half of its nodes are "large." If the new - * item is also "large" and falls on the half with - * "large" nodes, it also may not fit. - * - * As a final tweak, if the new item goes on the last - * spot on the page (and thus, onto the new page), bias - * the split so the new page is emptier than the old page. - * This yields better packing during sequential inserts. - */ - if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { - /* Find split point */ - psize = 0; - if (newindx <= split_indx || newindx >= nkeys) { - i = 0; j = 1; - k = newindx >= nkeys ? nkeys : split_indx+2; - } else { - i = nkeys; j = -1; - k = split_indx-1; - } - for (; i!=k; i+=j) { - if (i == newindx) { - psize += nsize; - node = NULL; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - psize += sizeof(pgno_t); - else - psize += NODEDSZ(node); - } - psize = EVEN(psize); - } - if (psize > pmax || i == k-j) { - split_indx = i + (j<0); - break; - } - } - } - if (split_indx == newindx) { - sepkey.mv_size = newkey->mv_size; - sepkey.mv_data = newkey->mv_data; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); - sepkey.mv_size = node->mn_ksize; - sepkey.mv_data = NODEKEY(node); - } - } - } - - DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); - - /* Copy separator key to the parent. - */ - if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { - mn.mc_snum--; - mn.mc_top--; - did_split = 1; - rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); - if (rc) - goto done; - - /* root split? */ - if (mn.mc_snum == mc->mc_snum) { - mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; - mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; - mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; - mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; - mc->mc_snum++; - mc->mc_top++; - ptop++; - } - /* Right page might now have changed parent. - * Check if left page also changed parent. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; imc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - mc->mc_pg[ptop] = mn.mc_pg[ptop]; - if (mn.mc_ki[ptop]) { - mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; - } else { - /* find right page's left sibling */ - mc->mc_ki[ptop] = mn.mc_ki[ptop]; - mdb_cursor_sibling(mc, 0); - } - } - } else { - mn.mc_top--; - rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); - mn.mc_top++; - } - mc->mc_flags ^= C_SPLITTING; - if (rc != MDB_SUCCESS) { - goto done; - } - if (nflags & MDB_APPEND) { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[mc->mc_top] = 0; - rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); - if (rc) - goto done; - for (i=0; imc_top; i++) - mc->mc_ki[i] = mn.mc_ki[i]; - } else if (!IS_LEAF2(mp)) { - /* Move nodes */ - mc->mc_pg[mc->mc_top] = rp; - i = split_indx; - j = 0; - do { - if (i == newindx) { - rkey.mv_data = newkey->mv_data; - rkey.mv_size = newkey->mv_size; - if (IS_LEAF(mp)) { - rdata = newdata; - } else - pgno = newpgno; - flags = nflags; - /* Update index for the new key. */ - mc->mc_ki[mc->mc_top] = j; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - rkey.mv_data = NODEKEY(node); - rkey.mv_size = node->mn_ksize; - if (IS_LEAF(mp)) { - xdata.mv_data = NODEDATA(node); - xdata.mv_size = NODEDSZ(node); - rdata = &xdata; - } else - pgno = NODEPGNO(node); - flags = node->mn_flags; - } - - if (!IS_LEAF(mp) && j == 0) { - /* First branch index doesn't need key data. */ - rkey.mv_size = 0; - } - - rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); - if (rc) - goto done; - if (i == nkeys) { - i = 0; - j = 0; - mc->mc_pg[mc->mc_top] = copy; - } else { - i++; - j++; - } - } while (i != split_indx); - - nkeys = NUMKEYS(copy); - for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; - mp->mp_lower = copy->mp_lower; - mp->mp_upper = copy->mp_upper; - memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), - env->me_psize - copy->mp_upper - PAGEBASE); - - /* reset back to original page */ - if (newindx < split_indx) { - mc->mc_pg[mc->mc_top] = mp; - if (nflags & MDB_RESERVE) { - node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (!(node->mn_flags & F_BIGDATA)) - newdata->mv_data = NODEDATA(node); - } - } else { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; i<=ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - } - } - } - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - int fixup = NUMKEYS(mp); - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc) - continue; - if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_flags & C_SPLITTING) - continue; - if (new_root) { - int k; - /* root split */ - for (k=m3->mc_top; k>=0; k--) { - m3->mc_ki[k+1] = m3->mc_ki[k]; - m3->mc_pg[k+1] = m3->mc_pg[k]; - } - if (m3->mc_ki[0] >= split_indx) { - m3->mc_ki[0] = 1; - } else { - m3->mc_ki[0] = 0; - } - m3->mc_pg[0] = mc->mc_pg[0]; - m3->mc_snum++; - m3->mc_top++; - } - if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) - m3->mc_ki[mc->mc_top]++; - if (m3->mc_ki[mc->mc_top] >= fixup) { - m3->mc_pg[mc->mc_top] = rp; - m3->mc_ki[mc->mc_top] -= fixup; - m3->mc_ki[ptop] = mn.mc_ki[ptop]; - } - } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && - m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { - m3->mc_ki[ptop]++; - } - } - } - DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); - -done: - if (copy) /* tmp page */ - mdb_page_free(env, copy); - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_put(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned int flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - - if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) - return EINVAL; - - mdb_cursor_init(&mc, txn, dbi, &mx); - return mdb_cursor_put(&mc, key, data, flags); -} - -#ifndef MDB_WBUF -#define MDB_WBUF (1024*1024) -#endif - - /** State needed for a compacting copy. */ -typedef struct mdb_copy { - pthread_mutex_t mc_mutex; - pthread_cond_t mc_cond; - char *mc_wbuf[2]; - char *mc_over[2]; - MDB_env *mc_env; - MDB_txn *mc_txn; - int mc_wlen[2]; - int mc_olen[2]; - pgno_t mc_next_pgno; - HANDLE mc_fd; - int mc_status; - volatile int mc_new; - int mc_toggle; - -} mdb_copy; - - /** Dedicated writer thread for compacting copy. */ -static THREAD_RET ESECT -mdb_env_copythr(void *arg) -{ - mdb_copy *my = arg; - char *ptr; - int toggle = 0, wsize, rc; -#ifdef _WIN32 - DWORD len; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - int len; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#endif - - pthread_mutex_lock(&my->mc_mutex); - my->mc_new = 0; - pthread_cond_signal(&my->mc_cond); - for(;;) { - while (!my->mc_new) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - if (my->mc_new < 0) { - my->mc_new = 0; - break; - } - my->mc_new = 0; - wsize = my->mc_wlen[toggle]; - ptr = my->mc_wbuf[toggle]; -again: - while (wsize > 0) { - DO_WRITE(rc, my->mc_fd, ptr, wsize, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - if (rc) { - my->mc_status = rc; - break; - } - /* If there's an overflow page tail, write it too */ - if (my->mc_olen[toggle]) { - wsize = my->mc_olen[toggle]; - ptr = my->mc_over[toggle]; - my->mc_olen[toggle] = 0; - goto again; - } - my->mc_wlen[toggle] = 0; - toggle ^= 1; - pthread_cond_signal(&my->mc_cond); - } - pthread_cond_signal(&my->mc_cond); - pthread_mutex_unlock(&my->mc_mutex); - return (THREAD_RET)0; -#undef DO_WRITE -} - - /** Tell the writer thread there's a buffer ready to write */ -static int ESECT -mdb_env_cthr_toggle(mdb_copy *my, int st) -{ - int toggle = my->mc_toggle ^ 1; - pthread_mutex_lock(&my->mc_mutex); - if (my->mc_status) { - pthread_mutex_unlock(&my->mc_mutex); - return my->mc_status; - } - while (my->mc_new == 1) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - my->mc_new = st; - my->mc_toggle = toggle; - pthread_cond_signal(&my->mc_cond); - pthread_mutex_unlock(&my->mc_mutex); - return 0; -} - - /** Depth-first tree traversal for compacting copy. */ -static int ESECT -mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) -{ - MDB_cursor mc; - MDB_txn *txn = my->mc_txn; - MDB_node *ni; - MDB_page *mo, *mp, *leaf; - char *buf, *ptr; - int rc, toggle; - unsigned int i; - - /* Empty DB, nothing to do */ - if (*pg == P_INVALID) - return MDB_SUCCESS; - - mc.mc_snum = 1; - mc.mc_top = 0; - mc.mc_txn = txn; - - rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); - if (rc) - return rc; - rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); - if (rc) - return rc; - - /* Make cursor pages writable */ - buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); - if (buf == NULL) - return ENOMEM; - - for (i=0; imc_env->me_psize); - mc.mc_pg[i] = (MDB_page *)ptr; - ptr += my->mc_env->me_psize; - } - - /* This is writable space for a leaf page. Usually not needed. */ - leaf = (MDB_page *)ptr; - - toggle = my->mc_toggle; - while (mc.mc_snum > 0) { - unsigned n; - mp = mc.mc_pg[mc.mc_top]; - n = NUMKEYS(mp); - - if (IS_LEAF(mp)) { - if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - rc = mdb_page_get(txn, pg, &omp, NULL); - if (rc) - goto done; - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - memcpy(mo, omp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno; - my->mc_next_pgno += omp->mp_pages; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (omp->mp_pages > 1) { - my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); - my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); - } else if (ni->mn_flags & F_SUBDATA) { - MDB_db db; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&db, NODEDATA(ni), sizeof(db)); - my->mc_toggle = toggle; - rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); - if (rc) - goto done; - toggle = my->mc_toggle; - memcpy(NODEDATA(ni), &db, sizeof(db)); - } - } - } - } else { - mc.mc_ki[mc.mc_top]++; - if (mc.mc_ki[mc.mc_top] < n) { - pgno_t pg; -again: - ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); - pg = NODEPGNO(ni); - rc = mdb_page_get(txn, pg, &mp, NULL); - if (rc) - goto done; - mc.mc_top++; - mc.mc_snum++; - mc.mc_ki[mc.mc_top] = 0; - if (IS_BRANCH(mp)) { - /* Whenever we advance to a sibling branch page, - * we must proceed all the way down to its first leaf. - */ - mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); - goto again; - } else - mc.mc_pg[mc.mc_top] = mp; - continue; - } - } - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - mdb_page_copy(mo, mp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno++; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (mc.mc_top) { - /* Update parent if there is one */ - ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); - SETPGNO(ni, mo->mp_pgno); - mdb_cursor_pop(&mc); - } else { - /* Otherwise we're done */ - *pg = mo->mp_pgno; - break; - } - } -done: - free(buf); - return rc; -} - - /** Copy environment with compaction. */ -static int ESECT -mdb_env_copyfd1(MDB_env *env, HANDLE fd) -{ - MDB_meta *mm; - MDB_page *mp; - mdb_copy my; - MDB_txn *txn = NULL; - pthread_t thr; - int rc; - -#ifdef _WIN32 - my.mc_mutex = CreateMutex(NULL, FALSE, NULL); - my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); - my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); - if (my.mc_wbuf[0] == NULL) - return errno; -#else - pthread_mutex_init(&my.mc_mutex, NULL); - pthread_cond_init(&my.mc_cond, NULL); -#ifdef HAVE_MEMALIGN - my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); - if (my.mc_wbuf[0] == NULL) - return errno; -#else - rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); - if (rc) - return rc; -#endif -#endif - memset(my.mc_wbuf[0], 0, MDB_WBUF*2); - my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; - my.mc_wlen[0] = 0; - my.mc_wlen[1] = 0; - my.mc_olen[0] = 0; - my.mc_olen[1] = 0; - my.mc_next_pgno = 2; - my.mc_status = 0; - my.mc_new = 1; - my.mc_toggle = 0; - my.mc_env = env; - my.mc_fd = fd; - THREAD_CREATE(thr, mdb_env_copythr, &my); - - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - return rc; - - mp = (MDB_page *)my.mc_wbuf[0]; - memset(mp, 0, 2*env->me_psize); - mp->mp_pgno = 0; - mp->mp_flags = P_META; - mm = (MDB_meta *)METADATA(mp); - mdb_env_init_meta0(env, mm); - mm->mm_address = env->me_metas[0]->mm_address; - - mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); - mp->mp_pgno = 1; - mp->mp_flags = P_META; - *(MDB_meta *)METADATA(mp) = *mm; - mm = (MDB_meta *)METADATA(mp); - - /* Count the number of free pages, subtract from lastpg to find - * number of active pages - */ - { - MDB_ID freecount = 0; - MDB_cursor mc; - MDB_val key, data; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - freecount += txn->mt_dbs[0].md_branch_pages + - txn->mt_dbs[0].md_leaf_pages + - txn->mt_dbs[0].md_overflow_pages; - - /* Set metapage 1 */ - mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; - mm->mm_dbs[1] = txn->mt_dbs[1]; - if (mm->mm_last_pg > 1) { - mm->mm_dbs[1].md_root = mm->mm_last_pg; - mm->mm_txnid = 1; - } else { - mm->mm_dbs[1].md_root = P_INVALID; - } - } - my.mc_wlen[0] = env->me_psize * 2; - my.mc_txn = txn; - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); - if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) - rc = mdb_env_cthr_toggle(&my, 1); - mdb_env_cthr_toggle(&my, -1); - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - THREAD_FINISH(thr); - - mdb_txn_abort(txn); -#ifdef _WIN32 - CloseHandle(my.mc_cond); - CloseHandle(my.mc_mutex); - _aligned_free(my.mc_wbuf[0]); -#else - pthread_cond_destroy(&my.mc_cond); - pthread_mutex_destroy(&my.mc_mutex); - free(my.mc_wbuf[0]); -#endif - return rc; -} - - /** Copy environment as-is. */ -static int ESECT -mdb_env_copyfd0(MDB_env *env, HANDLE fd) -{ - MDB_txn *txn = NULL; - int rc; - size_t wsize; - char *ptr; -#ifdef _WIN32 - DWORD len, w2; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - ssize_t len; - size_t w2; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#endif - - /* Do the lock/unlock of the reader mutex before starting the - * write txn. Otherwise other read txns could block writers. - */ - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - return rc; - - if (env->me_txns) { - /* We must start the actual read txn after blocking writers */ - mdb_txn_reset0(txn, "reset-stage1"); - - /* Temporarily block writers until we snapshot the meta pages */ - LOCK_MUTEX_W(env); - - rc = mdb_txn_renew0(txn); - if (rc) { - UNLOCK_MUTEX_W(env); - goto leave; - } - } - - wsize = env->me_psize * 2; - ptr = env->me_map; - w2 = wsize; - while (w2 > 0) { - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - w2 -= len; - continue; - } else { - /* Non-blocking or async handles are not supported */ - rc = EIO; - break; - } - } - if (env->me_txns) - UNLOCK_MUTEX_W(env); - - if (rc) - goto leave; - - w2 = txn->mt_next_pgno * env->me_psize; - { - size_t fsize = 0; - if ((rc = mdb_fsize(env->me_fd, &fsize))) - goto leave; - if (w2 > fsize) - w2 = fsize; - } - wsize = w2 - wsize; - while (wsize > 0) { - if (wsize > MAX_WRITE) - w2 = MAX_WRITE; - else - w2 = wsize; - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - -leave: - mdb_txn_abort(txn); - return rc; -} - -int ESECT -mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) -{ - if (flags & MDB_CP_COMPACT) - return mdb_env_copyfd1(env, fd); - else - return mdb_env_copyfd0(env, fd); -} - -int ESECT -mdb_env_copyfd(MDB_env *env, HANDLE fd) -{ - return mdb_env_copyfd2(env, fd, 0); -} - -int ESECT -mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) -{ - int rc, len; - char *lpath; - HANDLE newfd = INVALID_HANDLE_VALUE; - - if (env->me_flags & MDB_NOSUBDIR) { - lpath = (char *)path; - } else { - len = strlen(path); - len += sizeof(DATANAME); - lpath = malloc(len); - if (!lpath) - return ENOMEM; - sprintf(lpath, "%s" DATANAME, path); - } - - /* The destination path must exist, but the destination file must not. - * We don't want the OS to cache the writes, since the source data is - * already in the OS cache. - */ -#ifdef _WIN32 - newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, - FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); -#else - newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); -#endif - if (newfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if (env->me_psize >= env->me_os_psize) { -#ifdef O_DIRECT - /* Set O_DIRECT if the file system supports it */ - if ((rc = fcntl(newfd, F_GETFL)) != -1) - (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); -#endif -#ifdef F_NOCACHE /* __APPLE__ */ - rc = fcntl(newfd, F_NOCACHE, 1); - if (rc) { - rc = ErrCode(); - goto leave; - } -#endif - } - - rc = mdb_env_copyfd2(env, newfd, flags); - -leave: - if (!(env->me_flags & MDB_NOSUBDIR)) - free(lpath); - if (newfd != INVALID_HANDLE_VALUE) - if (close(newfd) < 0 && rc == MDB_SUCCESS) - rc = ErrCode(); - - return rc; -} - -int ESECT -mdb_env_copy(MDB_env *env, const char *path) -{ - return mdb_env_copy2(env, path, 0); -} - -int ESECT -mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) -{ - if ((flag & CHANGEABLE) != flag) - return EINVAL; - if (onoff) - env->me_flags |= flag; - else - env->me_flags &= ~flag; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_flags(MDB_env *env, unsigned int *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_flags; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_userctx(MDB_env *env, void *ctx) -{ - if (!env) - return EINVAL; - env->me_userctx = ctx; - return MDB_SUCCESS; -} - -void * ESECT -mdb_env_get_userctx(MDB_env *env) -{ - return env ? env->me_userctx : NULL; -} - -int ESECT -mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) -{ - if (!env) - return EINVAL; -#ifndef NDEBUG - env->me_assert_func = func; -#endif - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_path(MDB_env *env, const char **arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_path; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_fd; - return MDB_SUCCESS; -} - -/** Common code for #mdb_stat() and #mdb_env_stat(). - * @param[in] env the environment to operate in. - * @param[in] db the #MDB_db record containing the stats to return. - * @param[out] arg the address of an #MDB_stat structure to receive the stats. - * @return 0, this function always succeeds. - */ -static int ESECT -mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) -{ - arg->ms_psize = env->me_psize; - arg->ms_depth = db->md_depth; - arg->ms_branch_pages = db->md_branch_pages; - arg->ms_leaf_pages = db->md_leaf_pages; - arg->ms_overflow_pages = db->md_overflow_pages; - arg->ms_entries = db->md_entries; - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_stat(MDB_env *env, MDB_stat *arg) -{ - int toggle; - - if (env == NULL || arg == NULL) - return EINVAL; - - toggle = mdb_env_pick_meta(env); - - return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); -} - -int ESECT -mdb_env_info(MDB_env *env, MDB_envinfo *arg) -{ - int toggle; - - if (env == NULL || arg == NULL) - return EINVAL; - - toggle = mdb_env_pick_meta(env); - arg->me_mapaddr = env->me_metas[toggle]->mm_address; - arg->me_mapsize = env->me_mapsize; - arg->me_maxreaders = env->me_maxreaders; - - /* me_numreaders may be zero if this process never used any readers. Use - * the shared numreader count if it exists. - */ - arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; - - arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; - arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; - return MDB_SUCCESS; -} - -/** Set the default comparison functions for a database. - * Called immediately after a database is opened to set the defaults. - * The user can then override them with #mdb_set_compare() or - * #mdb_set_dupsort(). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -static void -mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) -{ - uint16_t f = txn->mt_dbs[dbi].md_flags; - - txn->mt_dbxs[dbi].md_cmp = - (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : - (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; - - txn->mt_dbxs[dbi].md_dcmp = - !(f & MDB_DUPSORT) ? 0 : - ((f & MDB_INTEGERDUP) - ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) - : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); -} - -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) -{ - MDB_val key, data; - MDB_dbi i; - MDB_cursor mc; - MDB_db dummy; - int rc, dbflag, exact; - unsigned int unused = 0, seq; - size_t len; - - if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, FREE_DBI); - } - - if ((flags & VALID_FLAGS) != flags) - return EINVAL; - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - /* main DB? */ - if (!name) { - *dbi = MAIN_DBI; - if (flags & PERSISTENT_FLAGS) { - uint16_t f2 = flags & PERSISTENT_FLAGS; - /* make sure flag changes get committed */ - if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { - txn->mt_dbs[MAIN_DBI].md_flags |= f2; - txn->mt_flags |= MDB_TXN_DIRTY; - } - } - mdb_default_cmp(txn, MAIN_DBI); - return MDB_SUCCESS; - } - - if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, MAIN_DBI); - } - - /* Is the DB already open? */ - len = strlen(name); - for (i=2; imt_numdbs; i++) { - if (!txn->mt_dbxs[i].md_name.mv_size) { - /* Remember this free slot */ - if (!unused) unused = i; - continue; - } - if (len == txn->mt_dbxs[i].md_name.mv_size && - !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { - *dbi = i; - return MDB_SUCCESS; - } - } - - /* If no free slot and max hit, fail */ - if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) - return MDB_DBS_FULL; - - /* Cannot mix named databases with some mainDB flags */ - if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) - return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; - - /* Find the DB info */ - dbflag = DB_NEW|DB_VALID; - exact = 0; - key.mv_size = len; - key.mv_data = (void *)name; - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); - if (rc == MDB_SUCCESS) { - /* make sure this is actually a DB */ - MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); - if (!(node->mn_flags & F_SUBDATA)) - return MDB_INCOMPATIBLE; - } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { - /* Create if requested */ - data.mv_size = sizeof(MDB_db); - data.mv_data = &dummy; - memset(&dummy, 0, sizeof(dummy)); - dummy.md_root = P_INVALID; - dummy.md_flags = flags & PERSISTENT_FLAGS; - rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); - dbflag |= DB_DIRTY; - } - - /* OK, got info, add to table */ - if (rc == MDB_SUCCESS) { - unsigned int slot = unused ? unused : txn->mt_numdbs; - txn->mt_dbxs[slot].md_name.mv_data = strdup(name); - txn->mt_dbxs[slot].md_name.mv_size = len; - txn->mt_dbxs[slot].md_rel = NULL; - txn->mt_dbflags[slot] = dbflag; - /* txn-> and env-> are the same in read txns, use - * tmp variable to avoid undefined assignment - */ - seq = ++txn->mt_env->me_dbiseqs[slot]; - txn->mt_dbiseqs[slot] = seq; - - memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); - *dbi = slot; - mdb_default_cmp(txn, slot); - if (!unused) { - txn->mt_numdbs++; - } - } - - return rc; -} - -int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) -{ - if (!arg || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_ERROR) - return MDB_BAD_TXN; - - if (txn->mt_dbflags[dbi] & DB_STALE) { - MDB_cursor mc; - MDB_xcursor mx; - /* Stale, must read the DB's root. cursor_init does it for us. */ - mdb_cursor_init(&mc, txn, dbi, &mx); - } - return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); -} - -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) -{ - char *ptr; - if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) - return; - ptr = env->me_dbxs[dbi].md_name.mv_data; - /* If there was no name, this was already closed */ - if (ptr) { - env->me_dbxs[dbi].md_name.mv_data = NULL; - env->me_dbxs[dbi].md_name.mv_size = 0; - env->me_dbflags[dbi] = 0; - env->me_dbiseqs[dbi]++; - free(ptr); - } -} - -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) -{ - /* We could return the flags for the FREE_DBI too but what's the point? */ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; - return MDB_SUCCESS; -} - -/** Add all the DB's pages to the free list. - * @param[in] mc Cursor on the DB to free. - * @param[in] subs non-Zero to check for sub-DBs in this DB. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_drop0(MDB_cursor *mc, int subs) -{ - int rc; - - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc == MDB_SUCCESS) { - MDB_txn *txn = mc->mc_txn; - MDB_node *ni; - MDB_cursor mx; - unsigned int i; - - /* LEAF2 pages have no nodes, cannot have sub-DBs */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) - mdb_cursor_pop(mc); - - mdb_cursor_copy(mc, &mx); - while (mc->mc_snum > 0) { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - unsigned n = NUMKEYS(mp); - if (IS_LEAF(mp)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - rc = mdb_page_get(txn, pg, &omp, NULL); - if (rc != 0) - goto done; - mdb_cassert(mc, IS_OVERFLOW(omp)); - rc = mdb_midl_append_range(&txn->mt_free_pgs, - pg, omp->mp_pages); - if (rc) - goto done; - } else if (subs && (ni->mn_flags & F_SUBDATA)) { - mdb_xcursor_init1(mc, ni); - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto done; - } - } - } else { - if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) - goto done; - for (i=0; imt_free_pgs, pg); - } - } - if (!mc->mc_top) - break; - mc->mc_ki[mc->mc_top] = i; - rc = mdb_cursor_sibling(mc, 1); - if (rc) { - if (rc != MDB_NOTFOUND) - goto done; - /* no more siblings, go back to beginning - * of previous level. - */ - mdb_cursor_pop(mc); - mc->mc_ki[0] = 0; - for (i=1; imc_snum; i++) { - mc->mc_ki[i] = 0; - mc->mc_pg[i] = mx.mc_pg[i]; - } - } - } - /* free it */ - rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); -done: - if (rc) - txn->mt_flags |= MDB_TXN_ERROR; - } else if (rc == MDB_NOTFOUND) { - rc = MDB_SUCCESS; - } - return rc; -} - -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) -{ - MDB_cursor *mc, *m2; - int rc; - - if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EACCES; - - if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) - return MDB_BAD_DBI; - - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) - return rc; - - rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); - /* Invalidate the dropped DB's cursors */ - for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) - m2->mc_flags &= ~(C_INITIALIZED|C_EOF); - if (rc) - goto leave; - - /* Can't delete the main DB */ - if (del && dbi > MAIN_DBI) { - rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); - if (!rc) { - txn->mt_dbflags[dbi] = DB_STALE; - mdb_dbi_close(txn->mt_env, dbi); - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - } else { - /* reset the DB record, mark it dirty */ - txn->mt_dbflags[dbi] |= DB_DIRTY; - txn->mt_dbs[dbi].md_depth = 0; - txn->mt_dbs[dbi].md_branch_pages = 0; - txn->mt_dbs[dbi].md_leaf_pages = 0; - txn->mt_dbs[dbi].md_overflow_pages = 0; - txn->mt_dbs[dbi].md_entries = 0; - txn->mt_dbs[dbi].md_root = P_INVALID; - - txn->mt_flags |= MDB_TXN_DIRTY; - } -leave: - mdb_cursor_close(mc); - return rc; -} - -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_cmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_dcmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_rel = rel; - return MDB_SUCCESS; -} - -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) -{ - if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) - return EINVAL; - - txn->mt_dbxs[dbi].md_relctx = ctx; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxkeysize(MDB_env *env) -{ - return ENV_MAXKEY(env); -} - -int ESECT -mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) -{ - unsigned int i, rdrs; - MDB_reader *mr; - char buf[64]; - int rc = 0, first = 1; - - if (!env || !func) - return -1; - if (!env->me_txns) { - return func("(no reader locks)\n", ctx); - } - rdrs = env->me_txns->mti_numreaders; - mr = env->me_txns->mti_readers; - for (i=0; i> 1; - cursor = base + pivot + 1; - val = pid - ids[cursor]; - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - /* found, so it's a duplicate */ - return -1; - } - } - - if( val > 0 ) { - ++cursor; - } - ids[0]++; - for (n = ids[0]; n > cursor; n--) - ids[n] = ids[n-1]; - ids[n] = pid; - return 0; -} - -int ESECT -mdb_reader_check(MDB_env *env, int *dead) -{ - unsigned int i, j, rdrs; - MDB_reader *mr; - MDB_PID_T *pids, pid; - int count = 0; - - if (!env) - return EINVAL; - if (dead) - *dead = 0; - if (!env->me_txns) - return MDB_SUCCESS; - rdrs = env->me_txns->mti_numreaders; - pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); - if (!pids) - return ENOMEM; - pids[0] = 0; - mr = env->me_txns->mti_readers; - for (i=0; ime_pid) { - pid = mr[i].mr_pid; - if (mdb_pid_insert(pids, pid) == 0) { - if (!mdb_reader_pid(env, Pidcheck, pid)) { - LOCK_MUTEX_R(env); - /* Recheck, a new process may have reused pid */ - if (!mdb_reader_pid(env, Pidcheck, pid)) { - for (j=i; j. - * - * Copyright 2000-2015 The OpenLDAP Foundation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include -#include -#include -#include -#include "midl.h" - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup idls ID List Management - * @{ - */ -#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) - -unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = ids[0]; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( ids[cursor], id ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -#if 0 /* superseded by append/sort */ -int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) -{ - unsigned x, i; - - x = mdb_midl_search( ids, id ); - assert( x > 0 ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0] && ids[x] == id ) { - /* duplicate */ - assert(0); - return -1; - } - - if ( ++ids[0] >= MDB_IDL_DB_MAX ) { - /* no room */ - --ids[0]; - return -2; - - } else { - /* insert id */ - for (i=ids[0]; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = id; - } - - return 0; -} -#endif - -MDB_IDL mdb_midl_alloc(int num) -{ - MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); - if (ids) { - *ids++ = num; - *ids = 0; - } - return ids; -} - -void mdb_midl_free(MDB_IDL ids) -{ - if (ids) - free(ids-1); -} - -int mdb_midl_shrink( MDB_IDL *idp ) -{ - MDB_IDL ids = *idp; - if (*(--ids) > MDB_IDL_UM_MAX && - (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) - { - *ids++ = MDB_IDL_UM_MAX; - *idp = ids; - return 1; - } - return 0; -} - -static int mdb_midl_grow( MDB_IDL *idp, int num ) -{ - MDB_IDL idn = *idp-1; - /* grow it */ - idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); - if (!idn) - return ENOMEM; - *idn++ += num; - *idp = idn; - return 0; -} - -int mdb_midl_need( MDB_IDL *idp, unsigned num ) -{ - MDB_IDL ids = *idp; - num += ids[0]; - if (num > ids[-1]) { - num = (num + num/4 + (256 + 2)) & -256; - if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) - return ENOMEM; - *ids++ = num - 2; - *idp = ids; - } - return 0; -} - -int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] >= ids[-1]) { - if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0]++; - ids[ids[0]] = id; - return 0; -} - -int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] + app[0] >= ids[-1]) { - if (mdb_midl_grow(idp, app[0])) - return ENOMEM; - ids = *idp; - } - memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); - ids[0] += app[0]; - return 0; -} - -int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) -{ - MDB_ID *ids = *idp, len = ids[0]; - /* Too big? */ - if (len + n > ids[-1]) { - if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0] = len + n; - ids += len; - while (n) - ids[n--] = id++; - return 0; -} - -void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) -{ - MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; - idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ - old_id = idl[j]; - while (i) { - merge_id = merge[i--]; - for (; old_id < merge_id; old_id = idl[--j]) - idl[k--] = old_id; - idl[k--] = merge_id; - } - idl[0] = total; -} - -/* Quicksort + Insertion sort for small arrays */ - -#define SMALL 8 -#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } - -void -mdb_midl_sort( MDB_IDL ids ) -{ - /* Max possible depth of int-indexed tree * 2 items/level */ - int istack[sizeof(int)*CHAR_BIT * 2]; - int i,j,k,l,ir,jstack; - MDB_ID a, itmp; - - ir = (int)ids[0]; - l = 1; - jstack = 0; - for(;;) { - if (ir - l < SMALL) { /* Insertion sort */ - for (j=l+1;j<=ir;j++) { - a = ids[j]; - for (i=j-1;i>=1;i--) { - if (ids[i] >= a) break; - ids[i+1] = ids[i]; - } - ids[i+1] = a; - } - if (jstack == 0) break; - ir = istack[jstack--]; - l = istack[jstack--]; - } else { - k = (l + ir) >> 1; /* Choose median of left, center, right */ - MIDL_SWAP(ids[k], ids[l+1]); - if (ids[l] < ids[ir]) { - MIDL_SWAP(ids[l], ids[ir]); - } - if (ids[l+1] < ids[ir]) { - MIDL_SWAP(ids[l+1], ids[ir]); - } - if (ids[l] < ids[l+1]) { - MIDL_SWAP(ids[l], ids[l+1]); - } - i = l+1; - j = ir; - a = ids[l+1]; - for(;;) { - do i++; while(ids[i] > a); - do j--; while(ids[j] < a); - if (j < i) break; - MIDL_SWAP(ids[i],ids[j]); - } - ids[l+1] = ids[j]; - ids[j] = a; - jstack += 2; - if (ir-i+1 >= j-l) { - istack[jstack] = ir; - istack[jstack-1] = i; - ir = j-1; - } else { - istack[jstack] = j-1; - istack[jstack-1] = l; - l = i; - } - } - } -} - -unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = (unsigned)ids[0].mid; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( id, ids[cursor].mid ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) -{ - unsigned x, i; - - x = mdb_mid2l_search( ids, id->mid ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0].mid && ids[x].mid == id->mid ) { - /* duplicate */ - return -1; - } - - if ( ids[0].mid >= MDB_IDL_UM_MAX ) { - /* too big */ - return -2; - - } else { - /* insert id */ - ids[0].mid++; - for (i=(unsigned)ids[0].mid; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = *id; - } - - return 0; -} - -int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) -{ - /* Too big? */ - if (ids[0].mid >= MDB_IDL_UM_MAX) { - return -2; - } - ids[0].mid++; - ids[ids[0].mid] = *id; - return 0; -} - -/** @} */ -/** @} */ diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/midl.h stb-tester-31/vendor/py-lmdb/lib/midl.h --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/midl.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lib/midl.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,186 +0,0 @@ -/** @file midl.h - * @brief LMDB ID List header file. - * - * This file was originally part of back-bdb but has been - * modified for use in libmdb. Most of the macros defined - * in this file are unused, just left over from the original. - * - * This file is only used internally in libmdb and its definitions - * are not exposed publicly. - */ -/* $OpenLDAP$ */ -/* This work is part of OpenLDAP Software . - * - * Copyright 2000-2015 The OpenLDAP Foundation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#ifndef _MDB_MIDL_H_ -#define _MDB_MIDL_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ - -/** @defgroup idls ID List Management - * @{ - */ - /** A generic unsigned ID number. These were entryIDs in back-bdb. - * Preferably it should have the same size as a pointer. - */ -typedef size_t MDB_ID; - - /** An IDL is an ID List, a sorted array of IDs. The first - * element of the array is a counter for how many actual - * IDs are in the list. In the original back-bdb code, IDLs are - * sorted in ascending order. For libmdb IDLs are sorted in - * descending order. - */ -typedef MDB_ID *MDB_IDL; - -/* IDL sizes - likely should be even bigger - * limiting factors: sizeof(ID), thread stack size - */ -#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ -#define MDB_IDL_DB_SIZE (1<. - * - * OpenLDAP is a registered trademark of the OpenLDAP Foundation. - * - * Individual files and/or contributed packages may be copyright by - * other parties and/or subject to additional restrictions. - * - * This work also contains materials derived from public sources. - * - * Additional information about OpenLDAP can be obtained at - * . - */ - -#ifndef LMDB_PRELOAD_H -#define LMDB_PRELOAD_H - -/** - * Touch a byte from every page in `x`, causing any read faults necessary for - * copying the value to occur. This should be called with the GIL released, in - * order to dramatically decrease the chances of a page fault being taken with - * the GIL held. - * - * We do this since PyMalloc cannot be invoked with the GIL released, and we - * cannot know the size of the MDB result value before dropping the GIL. This - * seems the simplest and cheapest compromise to ensuring multithreaded Python - * apps don't hard stall when dealing with a database larger than RAM. - */ -static void preload(int rc, void *x, size_t size) { - if(rc == 0) { - volatile char j; - int i; - for(i = 0; i < size; i += 4096) { - j = ((volatile char *)x)[i]; - } - (void) j; /* -Wunused-variable */ - } -} - -#endif /* !LMDB_PRELOAD_H */ diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/win32/inttypes.h stb-tester-31/vendor/py-lmdb/lib/win32/inttypes.h --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/win32/inttypes.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lib/win32/inttypes.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,306 +0,0 @@ -// ISO C9x compliant inttypes.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_INTTYPES_H_ // [ -#define _MSC_INTTYPES_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#include "stdint.h" - -// 7.8 Format conversion of integer types - -typedef struct { - intmax_t quot; - intmax_t rem; -} imaxdiv_t; - -// 7.8.1 Macros for format specifiers - -#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 - -// The fprintf macros for signed integers are: -#define PRId8 "d" -#define PRIi8 "i" -#define PRIdLEAST8 "d" -#define PRIiLEAST8 "i" -#define PRIdFAST8 "d" -#define PRIiFAST8 "i" - -#define PRId16 "hd" -#define PRIi16 "hi" -#define PRIdLEAST16 "hd" -#define PRIiLEAST16 "hi" -#define PRIdFAST16 "hd" -#define PRIiFAST16 "hi" - -#define PRId32 "I32d" -#define PRIi32 "I32i" -#define PRIdLEAST32 "I32d" -#define PRIiLEAST32 "I32i" -#define PRIdFAST32 "I32d" -#define PRIiFAST32 "I32i" - -#define PRId64 "I64d" -#define PRIi64 "I64i" -#define PRIdLEAST64 "I64d" -#define PRIiLEAST64 "I64i" -#define PRIdFAST64 "I64d" -#define PRIiFAST64 "I64i" - -#define PRIdMAX "I64d" -#define PRIiMAX "I64i" - -#define PRIdPTR "Id" -#define PRIiPTR "Ii" - -// The fprintf macros for unsigned integers are: -#define PRIo8 "o" -#define PRIu8 "u" -#define PRIx8 "x" -#define PRIX8 "X" -#define PRIoLEAST8 "o" -#define PRIuLEAST8 "u" -#define PRIxLEAST8 "x" -#define PRIXLEAST8 "X" -#define PRIoFAST8 "o" -#define PRIuFAST8 "u" -#define PRIxFAST8 "x" -#define PRIXFAST8 "X" - -#define PRIo16 "ho" -#define PRIu16 "hu" -#define PRIx16 "hx" -#define PRIX16 "hX" -#define PRIoLEAST16 "ho" -#define PRIuLEAST16 "hu" -#define PRIxLEAST16 "hx" -#define PRIXLEAST16 "hX" -#define PRIoFAST16 "ho" -#define PRIuFAST16 "hu" -#define PRIxFAST16 "hx" -#define PRIXFAST16 "hX" - -#define PRIo32 "I32o" -#define PRIu32 "I32u" -#define PRIx32 "I32x" -#define PRIX32 "I32X" -#define PRIoLEAST32 "I32o" -#define PRIuLEAST32 "I32u" -#define PRIxLEAST32 "I32x" -#define PRIXLEAST32 "I32X" -#define PRIoFAST32 "I32o" -#define PRIuFAST32 "I32u" -#define PRIxFAST32 "I32x" -#define PRIXFAST32 "I32X" - -#define PRIo64 "I64o" -#define PRIu64 "I64u" -#define PRIx64 "I64x" -#define PRIX64 "I64X" -#define PRIoLEAST64 "I64o" -#define PRIuLEAST64 "I64u" -#define PRIxLEAST64 "I64x" -#define PRIXLEAST64 "I64X" -#define PRIoFAST64 "I64o" -#define PRIuFAST64 "I64u" -#define PRIxFAST64 "I64x" -#define PRIXFAST64 "I64X" - -#define PRIoMAX "I64o" -#define PRIuMAX "I64u" -#define PRIxMAX "I64x" -#define PRIXMAX "I64X" - -#define PRIoPTR "Io" -#define PRIuPTR "Iu" -#define PRIxPTR "Ix" -#define PRIXPTR "IX" - -// The fscanf macros for signed integers are: -#define SCNd8 "d" -#define SCNi8 "i" -#define SCNdLEAST8 "d" -#define SCNiLEAST8 "i" -#define SCNdFAST8 "d" -#define SCNiFAST8 "i" - -#define SCNd16 "hd" -#define SCNi16 "hi" -#define SCNdLEAST16 "hd" -#define SCNiLEAST16 "hi" -#define SCNdFAST16 "hd" -#define SCNiFAST16 "hi" - -#define SCNd32 "ld" -#define SCNi32 "li" -#define SCNdLEAST32 "ld" -#define SCNiLEAST32 "li" -#define SCNdFAST32 "ld" -#define SCNiFAST32 "li" - -#define SCNd64 "I64d" -#define SCNi64 "I64i" -#define SCNdLEAST64 "I64d" -#define SCNiLEAST64 "I64i" -#define SCNdFAST64 "I64d" -#define SCNiFAST64 "I64i" - -#define SCNdMAX "I64d" -#define SCNiMAX "I64i" - -#ifdef _WIN64 // [ -# define SCNdPTR "I64d" -# define SCNiPTR "I64i" -#else // _WIN64 ][ -# define SCNdPTR "ld" -# define SCNiPTR "li" -#endif // _WIN64 ] - -// The fscanf macros for unsigned integers are: -#define SCNo8 "o" -#define SCNu8 "u" -#define SCNx8 "x" -#define SCNX8 "X" -#define SCNoLEAST8 "o" -#define SCNuLEAST8 "u" -#define SCNxLEAST8 "x" -#define SCNXLEAST8 "X" -#define SCNoFAST8 "o" -#define SCNuFAST8 "u" -#define SCNxFAST8 "x" -#define SCNXFAST8 "X" - -#define SCNo16 "ho" -#define SCNu16 "hu" -#define SCNx16 "hx" -#define SCNX16 "hX" -#define SCNoLEAST16 "ho" -#define SCNuLEAST16 "hu" -#define SCNxLEAST16 "hx" -#define SCNXLEAST16 "hX" -#define SCNoFAST16 "ho" -#define SCNuFAST16 "hu" -#define SCNxFAST16 "hx" -#define SCNXFAST16 "hX" - -#define SCNo32 "lo" -#define SCNu32 "lu" -#define SCNx32 "lx" -#define SCNX32 "lX" -#define SCNoLEAST32 "lo" -#define SCNuLEAST32 "lu" -#define SCNxLEAST32 "lx" -#define SCNXLEAST32 "lX" -#define SCNoFAST32 "lo" -#define SCNuFAST32 "lu" -#define SCNxFAST32 "lx" -#define SCNXFAST32 "lX" - -#define SCNo64 "I64o" -#define SCNu64 "I64u" -#define SCNx64 "I64x" -#define SCNX64 "I64X" -#define SCNoLEAST64 "I64o" -#define SCNuLEAST64 "I64u" -#define SCNxLEAST64 "I64x" -#define SCNXLEAST64 "I64X" -#define SCNoFAST64 "I64o" -#define SCNuFAST64 "I64u" -#define SCNxFAST64 "I64x" -#define SCNXFAST64 "I64X" - -#define SCNoMAX "I64o" -#define SCNuMAX "I64u" -#define SCNxMAX "I64x" -#define SCNXMAX "I64X" - -#ifdef _WIN64 // [ -# define SCNoPTR "I64o" -# define SCNuPTR "I64u" -# define SCNxPTR "I64x" -# define SCNXPTR "I64X" -#else // _WIN64 ][ -# define SCNoPTR "lo" -# define SCNuPTR "lu" -# define SCNxPTR "lx" -# define SCNXPTR "lX" -#endif // _WIN64 ] - -#endif // __STDC_FORMAT_MACROS ] - -// 7.8.2 Functions for greatest-width integer types - -// 7.8.2.1 The imaxabs function -#define imaxabs _abs64 - -// 7.8.2.2 The imaxdiv function - -// This is modified version of div() function from Microsoft's div.c found -// in %MSVC.NET%\crt\src\div.c -#ifdef STATIC_IMAXDIV // [ -static -#else // STATIC_IMAXDIV ][ -_inline -#endif // STATIC_IMAXDIV ] -imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) -{ - imaxdiv_t result; - - result.quot = numer / denom; - result.rem = numer % denom; - - if (numer < 0 && result.rem > 0) { - // did division wrong; must fix up - ++result.quot; - result.rem -= denom; - } - - return result; -} - -// 7.8.2.3 The strtoimax and strtoumax functions -#define strtoimax _strtoi64 -#define strtoumax _strtoui64 - -// 7.8.2.4 The wcstoimax and wcstoumax functions -#define wcstoimax _wcstoi64 -#define wcstoumax _wcstoui64 - - -#endif // _MSC_INTTYPES_H_ ] diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/win32-stdint/stdint.h stb-tester-31/vendor/py-lmdb/lib/win32-stdint/stdint.h --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lib/win32-stdint/stdint.h 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lib/win32-stdint/stdint.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,259 +0,0 @@ -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#if _MSC_VER >= 1600 // [ -#include -#else // ] _MSC_VER >= 1600 [ - -#include - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus -extern "C" { -#endif -# include -#ifdef __cplusplus -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#endif // _MSC_VER >= 1600 ] - -#endif // _MSC_STDINT_H_ ] diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/LICENSE stb-tester-31/vendor/py-lmdb/LICENSE --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/LICENSE 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/LICENSE 1970-01-01 00:00:00.000000000 +0000 @@ -1,47 +0,0 @@ -The OpenLDAP Public License - Version 2.8, 17 August 2003 - -Redistribution and use of this software and associated documentation -("Software"), with or without modification, are permitted provided -that the following conditions are met: - -1. Redistributions in source form must retain copyright statements - and notices, - -2. Redistributions in binary form must reproduce applicable copyright - statements and notices, this list of conditions, and the following - disclaimer in the documentation and/or other materials provided - with the distribution, and - -3. Redistributions must contain a verbatim copy of this document. - -The OpenLDAP Foundation may revise this license from time to time. -Each revision is distinguished by a version number. You may use -this Software under terms of this license revision or under the -terms of any subsequent revision of the license. - -THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS -CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) -OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -The names of the authors and copyright holders must not be used in -advertising or otherwise to promote the sale, use or other dealing -in this Software without specific, written prior permission. Title -to copyright in this Software shall at all times remain with copyright -holders. - -OpenLDAP is a registered trademark of the OpenLDAP Foundation. - -Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, -California, USA. All Rights Reserved. Permission to copy and -distribute verbatim copies of this document is granted. diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/cffi.py stb-tester-31/vendor/py-lmdb/lmdb/cffi.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/cffi.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lmdb/cffi.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,2104 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -""" -CPython/CFFI wrapper for OpenLDAP's "Lightning" MDB database. - -Please see http://lmdb.readthedocs.org/ -""" - -from __future__ import absolute_import -from __future__ import with_statement - -import inspect -import os -import sys -import threading - -is_win32 = sys.platform == 'win32' -if is_win32: - import msvcrt - -try: - import __builtin__ -except ImportError: - import builtins as __builtin__ - -import lmdb -try: - from lmdb import _config -except ImportError: - _config = None - - -__all__ = ['Environment', 'Cursor', 'Transaction', 'open', - 'enable_drop_gil', 'version'] -__all__ += ['Error', 'KeyExistsError', 'NotFoundError', 'PageNotFoundError', - 'CorruptedError', 'PanicError', 'VersionMismatchError', - 'InvalidError', 'MapFullError', 'DbsFullError', 'ReadersFullError', - 'TlsFullError', 'TxnFullError', 'CursorFullError', 'PageFullError', - 'MapResizedError', 'IncompatibleError', 'BadRslotError', - 'BadTxnError', 'BadValsizeError', 'ReadonlyError', - 'InvalidParameterError', 'LockError', 'MemoryError', 'DiskError'] - - -# Handle moronic Python 3 mess. -UnicodeType = getattr(__builtin__, 'unicode', str) -BytesType = getattr(__builtin__, 'bytes', str) - -O_0755 = int('0755', 8) -O_0111 = int('0111', 8) -EMPTY_BYTES = UnicodeType().encode() - - -# Used to track context across CFFI callbcks. -_callbacks = threading.local() - -_CFFI_CDEF = ''' - typedef int mode_t; - typedef ... MDB_env; - typedef struct MDB_txn MDB_txn; - typedef struct MDB_cursor MDB_cursor; - typedef unsigned int MDB_dbi; - enum MDB_cursor_op { - MDB_FIRST, - MDB_FIRST_DUP, - MDB_GET_BOTH, - MDB_GET_BOTH_RANGE, - MDB_GET_CURRENT, - MDB_GET_MULTIPLE, - MDB_LAST, - MDB_LAST_DUP, - MDB_NEXT, - MDB_NEXT_DUP, - MDB_NEXT_MULTIPLE, - MDB_NEXT_NODUP, - MDB_PREV, - MDB_PREV_DUP, - MDB_PREV_NODUP, - MDB_SET, - MDB_SET_KEY, - MDB_SET_RANGE, - ... - }; - typedef enum MDB_cursor_op MDB_cursor_op; - - struct MDB_val { - size_t mv_size; - void *mv_data; - ...; - }; - typedef struct MDB_val MDB_val; - - struct MDB_stat { - unsigned int ms_psize; - unsigned int ms_depth; - size_t ms_branch_pages; - size_t ms_leaf_pages; - size_t ms_overflow_pages; - size_t ms_entries; - ...; - }; - typedef struct MDB_stat MDB_stat; - - struct MDB_envinfo { - void *me_mapaddr; - size_t me_mapsize; - size_t me_last_pgno; - size_t me_last_txnid; - unsigned int me_maxreaders; - unsigned int me_numreaders; - ...; - }; - typedef struct MDB_envinfo MDB_envinfo; - - typedef int (*MDB_cmp_func)(const MDB_val *a, const MDB_val *b); - typedef void (*MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, - void *relctx); - - char *mdb_strerror(int err); - int mdb_env_create(MDB_env **env); - int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, - mode_t mode); - int mdb_env_copy2(MDB_env *env, const char *path, int flags); - int mdb_env_copyfd2(MDB_env *env, int fd, int flags); - int mdb_env_stat(MDB_env *env, MDB_stat *stat); - int mdb_env_info(MDB_env *env, MDB_envinfo *stat); - int mdb_env_get_maxkeysize(MDB_env *env); - int mdb_env_sync(MDB_env *env, int force); - void mdb_env_close(MDB_env *env); - int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); - int mdb_env_get_flags(MDB_env *env, unsigned int *flags); - int mdb_env_get_path(MDB_env *env, const char **path); - int mdb_env_set_mapsize(MDB_env *env, size_t size); - int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); - int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); - int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); - int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, - MDB_txn **txn); - int mdb_txn_commit(MDB_txn *txn); - void mdb_txn_reset(MDB_txn *txn); - int mdb_txn_renew(MDB_txn *txn); - void mdb_txn_abort(MDB_txn *txn); - int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, - MDB_dbi *dbi); - int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); - int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del_); - int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); - void mdb_cursor_close(MDB_cursor *cursor); - int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); - int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); - int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val*data, int op); - - typedef int (MDB_msg_func)(const char *msg, void *ctx); - int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); - int mdb_reader_check(MDB_env *env, int *dead); - int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); - - #define MDB_VERSION_MAJOR ... - #define MDB_VERSION_MINOR ... - #define MDB_VERSION_PATCH ... - - #define EACCES ... - #define EAGAIN ... - #define EINVAL ... - #define ENOMEM ... - #define ENOSPC ... - - #define MDB_BAD_RSLOT ... - #define MDB_BAD_DBI ... - #define MDB_BAD_TXN ... - #define MDB_BAD_VALSIZE ... - #define MDB_CORRUPTED ... - #define MDB_CURSOR_FULL ... - #define MDB_DBS_FULL ... - #define MDB_INCOMPATIBLE ... - #define MDB_INVALID ... - #define MDB_KEYEXIST ... - #define MDB_MAP_FULL ... - #define MDB_MAP_RESIZED ... - #define MDB_NOTFOUND ... - #define MDB_PAGE_FULL ... - #define MDB_PAGE_NOTFOUND ... - #define MDB_PANIC ... - #define MDB_READERS_FULL ... - #define MDB_TLS_FULL ... - #define MDB_TXN_FULL ... - #define MDB_VERSION_MISMATCH ... - - #define MDB_APPEND ... - #define MDB_CREATE ... - #define MDB_DUPSORT ... - #define MDB_MAPASYNC ... - #define MDB_NODUPDATA ... - #define MDB_NOMETASYNC ... - #define MDB_NOOVERWRITE ... - #define MDB_NORDAHEAD ... - #define MDB_NOSUBDIR ... - #define MDB_NOSYNC ... - #define MDB_NOTLS ... - #define MDB_RDONLY ... - #define MDB_NOLOCK ... - #define MDB_REVERSEKEY ... - #define MDB_WRITEMAP ... - #define MDB_NOMEMINIT ... - #define MDB_CP_COMPACT ... - - // Helpers below inline MDB_vals. Avoids key alloc/dup on CPython, where - // CFFI will use PyString_AS_STRING when passed as an argument. - static int pymdb_del(MDB_txn *txn, MDB_dbi dbi, - char *key_s, size_t keylen, - char *val_s, size_t vallen); - static int pymdb_put(MDB_txn *txn, MDB_dbi dbi, - char *key_s, size_t keylen, - char *val_s, size_t vallen, - unsigned int flags); - static int pymdb_get(MDB_txn *txn, MDB_dbi dbi, - char *key_s, size_t keylen, - MDB_val *val_out); - static int pymdb_cursor_get(MDB_cursor *cursor, - char *key_s, size_t key_len, - char *data_s, size_t data_len, - MDB_val *key, MDB_val *data, int op); - static int pymdb_cursor_put(MDB_cursor *cursor, - char *key_s, size_t keylen, - char *val_s, size_t vallen, int flags); -''' - -_CFFI_VERIFY = ''' - #include - #include "lmdb.h" - #include "preload.h" - - // Helpers below inline MDB_vals. Avoids key alloc/dup on CPython, where - // CFFI will use PyString_AS_STRING when passed as an argument. - static int pymdb_get(MDB_txn *txn, MDB_dbi dbi, char *key_s, size_t keylen, - MDB_val *val_out) - { - MDB_val key = {keylen, key_s}; - int rc = mdb_get(txn, dbi, &key, val_out); - preload(rc, val_out->mv_data, val_out->mv_size); - return rc; - } - - static int pymdb_put(MDB_txn *txn, MDB_dbi dbi, char *key_s, size_t keylen, - char *val_s, size_t vallen, unsigned int flags) - { - MDB_val key = {keylen, key_s}; - MDB_val val = {vallen, val_s}; - return mdb_put(txn, dbi, &key, &val, flags); - } - - static int pymdb_del(MDB_txn *txn, MDB_dbi dbi, char *key_s, size_t keylen, - char *val_s, size_t vallen) - { - MDB_val key = {keylen, key_s}; - MDB_val val = {vallen, val_s}; - MDB_val *valptr; - if(vallen == 0) { - valptr = NULL; - } else { - valptr = &val; - } - return mdb_del(txn, dbi, &key, valptr); - } - - static int pymdb_cursor_get(MDB_cursor *cursor, - char *key_s, size_t key_len, - char *data_s, size_t data_len, - MDB_val *key, MDB_val *data, int op) - { - MDB_val tmp_key = {key_len, key_s}; - MDB_val tmp_data = {data_len, data_s}; - int rc = mdb_cursor_get(cursor, &tmp_key, &tmp_data, op); - if(! rc) { - preload(rc, tmp_data.mv_data, tmp_data.mv_size); - *key = tmp_key; - *data = tmp_data; - } - return rc; - } - - static int pymdb_cursor_put(MDB_cursor *cursor, char *key_s, size_t keylen, - char *val_s, size_t vallen, int flags) - { - MDB_val tmpkey = {keylen, key_s}; - MDB_val tmpval = {vallen, val_s}; - return mdb_cursor_put(cursor, &tmpkey, &tmpval, flags); - } -''' - -if not lmdb._reading_docs(): - import cffi - - # Try to use distutils-bundled CFFI configuration to avoid a recompile and - # potential compile errors during first module import. - _config_vars = _config.CONFIG if _config else { - 'extra_compile_args': ['-w'], - 'extra_sources': ['lib/mdb.c', 'lib/midl.c'], - 'extra_include_dirs': ['lib'], - 'extra_library_dirs': [], - 'libraries': [] - } - - _ffi = cffi.FFI() - _ffi.cdef(_CFFI_CDEF) - _lib = _ffi.verify(_CFFI_VERIFY, - modulename='lmdb_cffi', - ext_package='lmdb', - sources=_config_vars['extra_sources'], - extra_compile_args=_config_vars['extra_compile_args'], - include_dirs=_config_vars['extra_include_dirs'], - libraries=_config_vars['libraries'], - library_dirs=_config_vars['extra_library_dirs']) - - @_ffi.callback("int(char *, void *)") - def _msg_func(s, _): - """mdb_msg_func() callback. Appends `s` to _callbacks.msg_func list. - """ - _callbacks.msg_func.append(_ffi.string(s).decode()) - return 0 - -class Error(Exception): - """Raised when an LMDB-related error occurs, and no more specific - :py:class:`lmdb.Error` subclass exists.""" - def __init__(self, what, code=0): - self.what = what - self.code = code - self.reason = _ffi.string(_lib.mdb_strerror(code)) - msg = what - if code: - msg = '%s: %s' % (what, self.reason) - hint = getattr(self, 'MDB_HINT', None) - if hint: - msg += ' (%s)' % (hint,) - Exception.__init__(self, msg) - -class KeyExistsError(Error): - """Key/data pair already exists.""" - MDB_NAME = 'MDB_KEYEXIST' - -class NotFoundError(Error): - """No matching key/data pair found.""" - MDB_NAME = 'MDB_NOTFOUND' - -class PageNotFoundError(Error): - """Request page not found.""" - MDB_NAME = 'MDB_PAGE_NOTFOUND' - -class CorruptedError(Error): - """Located page was of the wrong type.""" - MDB_NAME = 'MDB_CORRUPTED' - -class PanicError(Error): - """Update of meta page failed.""" - MDB_NAME = 'MDB_PANIC' - -class VersionMismatchError(Error): - """Database environment version mismatch.""" - MDB_NAME = 'MDB_VERSION_MISMATCH' - -class InvalidError(Error): - """File is not an MDB file.""" - MDB_NAME = 'MDB_INVALID' - -class MapFullError(Error): - """Environment map_size= limit reached.""" - MDB_NAME = 'MDB_MAP_FULL' - MDB_HINT = 'Please use a larger Environment(map_size=) parameter' - -class DbsFullError(Error): - """Environment max_dbs= limit reached.""" - MDB_NAME = 'MDB_DBS_FULL' - MDB_HINT = 'Please use a larger Environment(max_dbs=) parameter' - -class ReadersFullError(Error): - """Environment max_readers= limit reached.""" - MDB_NAME = 'MDB_READERS_FULL' - MDB_HINT = 'Please use a larger Environment(max_readers=) parameter' - -class TlsFullError(Error): - """Thread-local storage keys full - too many environments open.""" - MDB_NAME = 'MDB_TLS_FULL' - -class TxnFullError(Error): - """Transaciton has too many dirty pages - transaction too big.""" - MDB_NAME = 'MDB_TXN_FULL' - MDB_HINT = 'Please do less work within your transaction' - -class CursorFullError(Error): - """Internal error - cursor stack limit reached.""" - MDB_NAME = 'MDB_CURSOR_FULL' - -class PageFullError(Error): - """Internal error - page has no more space.""" - MDB_NAME = 'MDB_PAGE_FULL' - -class MapResizedError(Error): - """Database contents grew beyond environment map_size=.""" - MDB_NAME = 'MDB_MAP_RESIZED' - -class IncompatibleError(Error): - """Operation and DB incompatible, or DB flags changed.""" - MDB_NAME = 'MDB_INCOMPATIBLE' - -class BadRslotError(Error): - """Invalid reuse of reader locktable slot.""" - MDB_NAME = 'MDB_BAD_RSLOT' - -class BadDbiError(Error): - """The specified DBI was changed unexpectedly.""" - MDB_NAME = 'MDB_BAD_DBI' - -class BadTxnError(Error): - """Transaction cannot recover - it must be aborted.""" - MDB_NAME = 'MDB_BAD_TXN' - -class BadValsizeError(Error): - """Too big key/data, key is empty, or wrong DUPFIXED size.""" - MDB_NAME = 'MDB_BAD_VALSIZE' - -class ReadonlyError(Error): - """An attempt was made to modify a read-only database.""" - MDB_NAME = 'EACCES' - -class InvalidParameterError(Error): - """An invalid parameter was specified.""" - MDB_NAME = 'EINVAL' - -class LockError(Error): - """The environment was locked by another process.""" - MDB_NAME = 'EAGAIN' - -class MemoryError(Error): - """Out of memory.""" - MDB_NAME = 'ENOMEM' - -class DiskError(Error): - """No more disk space.""" - MDB_NAME = 'ENOSPC' - -# Prepare _error_map, a mapping of integer MDB_ERROR_CODE to exception class. -if not lmdb._reading_docs(): - _error_map = {} - for obj in list(globals().values()): - if inspect.isclass(obj) and issubclass(obj, Error) and obj is not Error: - _error_map[getattr(_lib, obj.MDB_NAME)] = obj - del obj - -def _error(what, rc): - """Lookup and instantiate the correct exception class for the error code - `rc`, using :py:class:`Error` if no better class exists.""" - return _error_map.get(rc, Error)(what, rc) - -class Some_LMDB_Resource_That_Was_Deleted_Or_Closed(object): - """We need this because CFFI on PyPy treats None as cffi.NULL, instead of - throwing an exception it feeds LMDB null pointers. That means simply - replacing native handles with None during _invalidate() will cause NULL - pointer dereferences. Instead use this class, and its weird name to cause a - TypeError, with a very obvious string in the exception text. - - The only alternatives to this are inserting a check around every single use - of a native handle to ensure the handle is still valid prior to calling - LMDB, or doing no crash-safety checking at all. - """ - def __nonzero__(self): - return 0 - def __bool__(self): - return False - def __repr__(self): - return "" -_invalid = Some_LMDB_Resource_That_Was_Deleted_Or_Closed() - -def _mvbuf(mv): - """Convert a MDB_val cdata to a CFFI buffer object.""" - return _ffi.buffer(mv.mv_data, mv.mv_size) - -def _mvstr(mv): - """Convert a MDB_val cdata to Python bytes.""" - return _ffi.buffer(mv.mv_data, mv.mv_size)[:] - -def enable_drop_gil(): - """Deprecated.""" - -def version(): - """ - Return a tuple of integers `(major, minor, patch)` describing the LMDB - library version that the binding is linked against. The version of the - binding itself is available from ``lmdb.__version__``. - """ - return (_lib.MDB_VERSION_MAJOR, \ - _lib.MDB_VERSION_MINOR, \ - _lib.MDB_VERSION_PATCH) - - -class Environment(object): - """ - Structure for a database environment. An environment may contain multiple - databases, all residing in the same shared-memory map and underlying disk - file. - - To write to the environment a :py:class:`Transaction` must be created. One - simultaneous write transaction is allowed, however there is no limit on the - number of read transactions even when a write transaction exists. - - Equivalent to `mdb_env_open() - `_ - - `path`: - Location of directory (if `subdir=True`) or file prefix to store - the database. - - `map_size`: - Maximum size database may grow to; used to size the memory mapping. - If database grows larger than ``map_size``, an exception will be - raised and the user must close and reopen :py:class:`Environment`. - On 64-bit there is no penalty for making this huge (say 1TB). Must - be <2GB on 32-bit. - - .. note:: - - **The default map size is set low to encourage a crash**, so - users can figure out a good value before learning about this - option too late. - - `subdir`: - If ``True``, `path` refers to a subdirectory to store the data and - lock files in, otherwise it refers to a filename prefix. - - `readonly`: - If ``True``, disallow any write operations. Note the lock file is - still modified. If specified, the ``write`` flag to - :py:meth:`begin` or :py:class:`Transaction` is ignored. - - `metasync`: - If ``False``, flush system buffers to disk only once per - transaction, omit the metadata flush. Defer that until the system - flushes files to disk, or next commit or :py:meth:`sync`. - - This optimization maintains database integrity, but a system crash - may undo the last committed transaction. I.e. it preserves the ACI - (atomicity, consistency, isolation) but not D (durability) database - property. - - `sync`: - If ``False``, don't flush system buffers to disk when committing a - transaction. This optimization means a system crash can corrupt the - database or lose the last transactions if buffers are not yet - flushed to disk. - - The risk is governed by how often the system flushes dirty buffers - to disk and how often :py:meth:`sync` is called. However, if the - filesystem preserves write order and `writemap=False`, transactions - exhibit ACI (atomicity, consistency, isolation) properties and only - lose D (durability). I.e. database integrity is maintained, but a - system crash may undo the final transactions. - - Note that `sync=False, writemap=True` leaves the system with no - hint for when to write transactions to disk, unless :py:meth:`sync` - is called. `map_async=True, writemap=True` may be preferable. - - `mode`: - File creation mode. - - `create`: - If ``False``, do not create the directory `path` if it is missing. - - `readahead`: - If ``False``, LMDB will disable the OS filesystem readahead - mechanism, which may improve random read performance when a - database is larger than RAM. - - `writemap`: - If ``True``, use a writeable memory map unless `readonly=True`. - This is faster and uses fewer mallocs, but loses protection from - application bugs like wild pointer writes and other bad updates - into the database. Incompatible with nested transactions. - - Processes with and without `writemap` on the same environment do - not cooperate well. - - `meminit`: - If ``False`` LMDB will not zero-initialize buffers prior to writing - them to disk. This improves performance but may cause old heap data - to be written saved in the unused portion of the buffer. Do not use - this option if your application manipulates confidential data (e.g. - plaintext passwords) in memory. This option is only meaningful when - `writemap=False`; new pages are always zero-initialized when - `writemap=True`. - - `map_async`: - When ``writemap=True``, use asynchronous flushes to disk. As with - ``sync=False``, a system crash can then corrupt the database or - lose the last transactions. Calling :py:meth:`sync` ensures - on-disk database integrity until next commit. - - `max_readers`: - Maximum number of simultaneous read transactions. Can only be set - by the first process to open an environment, as it affects the size - of the lock file and shared memory area. Attempts to simultaneously - start more than this many *read* transactions will fail. - - `max_dbs`: - Maximum number of databases available. If 0, assume environment - will be used as a single database. - - `max_spare_txns`: - Read-only transactions to cache after becoming unused. Caching - transactions avoids two allocations, one lock and linear scan - of the shared environment per invocation of :py:meth:`begin`, - :py:class:`Transaction`, :py:meth:`get`, :py:meth:`gets`, or - :py:meth:`cursor`. Should match the process's maximum expected - concurrent transactions (e.g. thread count). - - `lock`: - If ``False``, don't do any locking. If concurrent access is - anticipated, the caller must manage all concurrency itself. For - proper operation the caller must enforce single-writer semantics, - and must ensure that no readers are using old transactions while a - writer is active. The simplest approach is to use an exclusive lock - so that no readers may be active at all when a writer begins. - """ - def __init__(self, path, map_size=10485760, subdir=True, - readonly=False, metasync=True, sync=True, map_async=False, - mode=O_0755, create=True, readahead=True, writemap=False, - meminit=True, max_readers=126, max_dbs=0, max_spare_txns=1, - lock=True): - self._max_spare_txns = max_spare_txns - self._spare_txns = [] - - envpp = _ffi.new('MDB_env **') - - rc = _lib.mdb_env_create(envpp) - if rc: - raise _error("mdb_env_create", rc) - self._env = envpp[0] - self._deps = set() - - rc = _lib.mdb_env_set_mapsize(self._env, map_size) - if rc: - raise _error("mdb_env_set_mapsize", rc) - - rc = _lib.mdb_env_set_maxreaders(self._env, max_readers) - if rc: - raise _error("mdb_env_set_maxreaders", rc) - - rc = _lib.mdb_env_set_maxdbs(self._env, max_dbs) - if rc: - raise _error("mdb_env_set_maxdbs", rc) - - if create and subdir and not (os.path.exists(path) or readonly): - os.mkdir(path, mode) - - flags = _lib.MDB_NOTLS - if not subdir: - flags |= _lib.MDB_NOSUBDIR - if readonly: - flags |= _lib.MDB_RDONLY - self.readonly = readonly - if not metasync: - flags |= _lib.MDB_NOMETASYNC - if not sync: - flags |= _lib.MDB_NOSYNC - if map_async: - flags |= _lib.MDB_MAPASYNC - if not readahead: - flags |= _lib.MDB_NORDAHEAD - if writemap: - flags |= _lib.MDB_WRITEMAP - if not meminit: - flags |= _lib.MDB_NOMEMINIT - if not lock: - flags |= _lib.MDB_NOLOCK - - if isinstance(path, UnicodeType): - path = path.encode(sys.getfilesystemencoding()) - - rc = _lib.mdb_env_open(self._env, path, flags, mode & ~O_0111) - if rc: - raise _error(path, rc) - - with self.begin(db=object()) as txn: - self._db = _Database(self, txn, None, False, False, True) - self._dbs = {None: self._db} - - def __enter__(self): - return self - - def __exit__(self, _1, _2, _3): - self.close() - - def __del__(self): - self.close() - - _env = None - _deps = None - _spare_txns = None - _dbs = None - - def close(self): - """Close the environment, invalidating any open iterators, cursors, and - transactions. Repeat calls to :py:meth:`close` have no effect. - - Equivalent to `mdb_env_close() - `_ - """ - if self._env: - if self._deps: - while self._deps: - self._deps.pop()._invalidate() - self._deps = None - - if self._spare_txns: - while self._spare_txns: - _lib.mdb_txn_abort(self._spare_txns.pop()) - self._spare_txns = None - - if self._dbs: - self._dbs.clear() - self._dbs = None - self._db = None - - _lib.mdb_env_close(self._env) - self._env = _invalid - - def path(self): - """Directory path or file name prefix where this environment is - stored. - - Equivalent to `mdb_env_get_path() - `_ - """ - path = _ffi.new('char **') - rc = _lib.mdb_env_get_path(self._env, path) - if rc: - raise _error("mdb_env_get_path", rc) - return _ffi.string(path[0]).decode(sys.getfilesystemencoding()) - - def copy(self, path, compact=False): - """Make a consistent copy of the environment in the given destination - directory. - - `compact`: - If ``True``, perform compaction while copying: omit free pages and - sequentially renumber all pages in output. This option consumes - more CPU and runs more slowly than the default, but may produce a - smaller output database. - - Equivalent to `mdb_env_copy() - `_ - """ - flags = _lib.MDB_CP_COMPACT if compact else 0 - encoded = path.encode(sys.getfilesystemencoding()) - rc = _lib.mdb_env_copy2(self._env, encoded, flags) - if rc: - raise _error("mdb_env_copy2", rc) - - def copyfd(self, fd, compact=False): - """Copy a consistent version of the environment to file descriptor - `fd`. - - `compact`: - If ``True``, perform compaction while copying: omit free pages and - sequentially renumber all pages in output. This option consumes - more CPU and runs more slowly than the default, but may produce a - smaller output database. - - Equivalent to `mdb_env_copyfd() - `_ - """ - if is_win32: - # Convert C library handle to kernel handle. - fd = msvcrt.get_osfhandle(fd) - flags = _lib.MDB_CP_COMPACT if compact else 0 - rc = _lib.mdb_env_copyfd2(self._env, fd, flags) - if rc: - raise _error("mdb_env_copyfd2", rc) - - def sync(self, force=False): - """Flush the data buffers to disk. - - Equivalent to `mdb_env_sync() - `_ - - Data is always written to disk when :py:meth:`Transaction.commit` is - called, but the operating system may keep it buffered. MDB always - flushes the OS buffers upon commit as well, unless the environment was - opened with `sync=False` or `metasync=False`. - - `force`: - If ``True``, force a synchronous flush. Otherwise if the - environment was opened with `sync=False` the flushes will be - omitted, and with `map_async=True` they will be asynchronous. - """ - rc = _lib.mdb_env_sync(self._env, force) - if rc: - raise _error("mdb_env_sync", rc) - - def _convert_stat(self, st): - """Convert a MDB_stat to a dict. - """ - return { - "psize": st.ms_psize, - "depth": st.ms_depth, - "branch_pages": st.ms_branch_pages, - "leaf_pages": st.ms_leaf_pages, - "overflow_pages": st.ms_overflow_pages, - "entries": st.ms_entries - } - - def stat(self): - """stat() - - Return some nice environment statistics as a dict: - - +--------------------+---------------------------------------+ - | ``psize`` | Size of a database page in bytes. | - +--------------------+---------------------------------------+ - | ``depth`` | Height of the B-tree. | - +--------------------+---------------------------------------+ - | ``branch_pages`` | Number of internal (non-leaf) pages. | - +--------------------+---------------------------------------+ - | ``leaf_pages`` | Number of leaf pages. | - +--------------------+---------------------------------------+ - | ``overflow_pages`` | Number of overflow pages. | - +--------------------+---------------------------------------+ - | ``entries`` | Number of data items. | - +--------------------+---------------------------------------+ - - Equivalent to `mdb_env_stat() - `_ - """ - st = _ffi.new('MDB_stat *') - rc = _lib.mdb_env_stat(self._env, st) - if rc: - raise _error("mdb_env_stat", rc) - return self._convert_stat(st) - - def info(self): - """Return some nice environment information as a dict: - - +--------------------+---------------------------------------------+ - | ``map_addr`` | Address of database map in RAM. | - +--------------------+---------------------------------------------+ - | ``map_size`` | Size of database map in RAM. | - +--------------------+---------------------------------------------+ - | ``last_pgno`` | ID of last used page. | - +--------------------+---------------------------------------------+ - | ``last_txnid`` | ID of last committed transaction. | - +--------------------+---------------------------------------------+ - | ``max_readers`` | Number of reader slots allocated in the | - | | lock file. Equivalent to the value of | - | | `maxreaders=` specified by the first | - | | process opening the Environment. | - +--------------------+---------------------------------------------+ - | ``num_readers`` | Maximum number of reader slots in | - | | simultaneous use since the lock file was | - | | initialized. | - +--------------------+---------------------------------------------+ - - Equivalent to `mdb_env_info() - `_ - """ - info = _ffi.new('MDB_envinfo *') - rc = _lib.mdb_env_info(self._env, info) - if rc: - raise _error("mdb_env_info", rc) - return { - "map_addr": int(_ffi.cast('long', info.me_mapaddr)), - "map_size": info.me_mapsize, - "last_pgno": info.me_last_pgno, - "last_txnid": info.me_last_txnid, - "max_readers": info.me_maxreaders, - "num_readers": info.me_numreaders - } - - def flags(self): - """Return a dict describing Environment constructor flags used to - instantiate this environment.""" - flags_ = _ffi.new('unsigned int[]', 1) - rc = _lib.mdb_env_get_flags(self._env, flags_) - if rc: - raise _error("mdb_env_get_flags", rc) - flags = flags_[0] - return { - 'subdir': not (flags & _lib.MDB_NOSUBDIR), - 'readonly': bool(flags & _lib.MDB_RDONLY), - 'metasync': not (flags & _lib.MDB_NOMETASYNC), - 'sync': not (flags & _lib.MDB_NOSYNC), - 'map_async': bool(flags & _lib.MDB_MAPASYNC), - 'readahead': not (flags & _lib.MDB_NORDAHEAD), - 'writemap': bool(flags & _lib.MDB_WRITEMAP), - 'meminit': not (flags & _lib.MDB_NOMEMINIT), - 'lock': not (flags & _lib.MDB_NOLOCK), - } - - def max_key_size(self): - """Return the maximum size in bytes of a record's key part. This - matches the ``MDB_MAXKEYSIZE`` constant set at compile time.""" - return _lib.mdb_env_get_maxkeysize(self._env) - - def max_readers(self): - """Return the maximum number of readers specified during open of the - environment by the first process. This is the same as `max_readers=` - specified to the constructor if this process was the first to open the - environment.""" - readers_ = _ffi.new('unsigned int[]', 1) - rc = _lib.mdb_env_get_maxreaders(self._env, readers_) - if rc: - raise _error("mdb_env_get_maxreaders", rc) - return readers_[0] - - def readers(self): - """Return a multi line Unicode string describing the current state of - the reader lock table.""" - _callbacks.msg_func = [] - try: - rc = _lib.mdb_reader_list(self._env, _msg_func, _ffi.NULL) - if rc: - raise _error("mdb_reader_list", rc) - return UnicodeType().join(_callbacks.msg_func) - finally: - del _callbacks.msg_func - - def reader_check(self): - """Search the reader lock table for stale entries, for example due to a - crashed process. Returns the number of stale entries that were cleared. - """ - reaped = _ffi.new('int[]', 1) - rc = _lib.mdb_reader_check(self._env, reaped) - if rc: - raise _error('mdb_reader_check', rc) - return reaped[0] - - def open_db(self, key=None, txn=None, reverse_key=False, dupsort=False, - create=True): - """ - Open a database, returning an opaque handle. Repeat - :py:meth:`Environment.open_db` calls for the same name will return the - same handle. As a special case, the main database is always open. - - Equivalent to `mdb_dbi_open() - `_ - - Named databases are implemented by *storing a special descriptor in the - main database*. All databases in an environment *share the same file*. - Because the descriptor is present in the main database, attempts to - create a named database will fail if a key matching the database's name - already exists. Furthermore *the key is visible to lookups and - enumerations*. If your main database keyspace conflicts with the names - you use for named databases, then move the contents of your main - database to another named database. - - :: - - >>> env = lmdb.open('/tmp/test', max_dbs=2) - >>> with env.begin(write=True) as txn - ... txn.put('somename', 'somedata') - - >>> # Error: database cannot share name of existing key! - >>> subdb = env.open_db('somename') - - A newly created database will not exist if the transaction that created - it aborted, nor if another process deleted it. The handle resides in - the shared environment, it is not owned by the current transaction or - process. Only one thread should call this function; it is not - mutex-protected in a read-only transaction. - - Preexisting transactions, other than the current transaction and any - parents, must not use the new handle, nor must their children. - - `key`: - Bytestring database name. If ``None``, indicates the main - database should be returned, otherwise indicates a named - database should be created inside the main database. - - In other words, *a key representing the database will be - visible in the main database, and the database name cannot - conflict with any existing key.* - - `txn`: - Transaction used to create the database if it does not exist. - If unspecified, a temporarily write transaction is used. Do not - call :py:meth:`open_db` from inside an existing transaction - without supplying it here. Note the passed transaction must - have `write=True`. - - `reverse_key`: - If ``True``, keys are compared from right to left (e.g. DNS - names). - - `dupsort`: - Duplicate keys may be used in the database. (Or, from another - perspective, keys may have multiple data items, stored in - sorted order.) By default keys must be unique and may have only - a single data item. - - `create`: - If ``True``, create the database if it doesn't exist, otherwise - raise an exception. - """ - if isinstance(key, UnicodeType): - raise TypeError('key must be bytes') - - db = self._dbs.get(key) - if db: - return db - - if txn: - db = _Database(self, txn, key, reverse_key, dupsort, create) - else: - with self.begin(write=True) as txn: - db = _Database(self, txn, key, reverse_key, dupsort, create) - self._dbs[key] = db - return db - - def begin(self, db=None, parent=None, write=False, buffers=False): - """Shortcut for :py:class:`lmdb.Transaction`""" - return Transaction(self, db, parent, write, buffers) - - -class _Database(object): - """Internal database handle.""" - def __init__(self, env, txn, name, reverse_key, dupsort, create): - env._deps.add(self) - self._deps = set() - - flags = 0 - if reverse_key: - flags |= _lib.MDB_REVERSEKEY - if dupsort: - flags |= _lib.MDB_DUPSORT - if create: - flags |= _lib.MDB_CREATE - dbipp = _ffi.new('MDB_dbi *') - self._dbi = None - rc = _lib.mdb_dbi_open(txn._txn, name or _ffi.NULL, flags, dbipp) - if rc: - raise _error("mdb_dbi_open", rc) - self._dbi = dbipp[0] - self._load_flags(txn) - - def _load_flags(self, txn): - """Load MDB's notion of the database flags.""" - flags_ = _ffi.new('unsigned int[]', 1) - rc = _lib.mdb_dbi_flags(txn._txn, self._dbi, flags_) - if rc: - raise _error("mdb_dbi_flags", rc) - self._flags = flags_[0] - - def flags(self, txn): - """Return the database's associated flags as a dict of _Database - constructor kwargs.""" - return { - 'reverse_key': bool(self._flags & _lib.MDB_REVERSEKEY), - 'dupsort': bool(self._flags & _lib.MDB_DUPSORT), - } - - def _invalidate(self): - self._dbi = _invalid - -open = Environment - - -class Transaction(object): - """ - A transaction object. All operations require a transaction handle, - transactions may be read-only or read-write. Write transactions may not - span threads. Transaction objects implement the context manager protocol, - so that reliable release of the transaction happens even in the face of - unhandled exceptions: - - .. code-block:: python - - # Transaction aborts correctly: - with env.begin(write=True) as txn: - crash() - - # Transaction commits automatically: - with env.begin(write=True) as txn: - txn.put('a', 'b') - - Equivalent to `mdb_txn_begin() - `_ - - `env`: - Environment the transaction should be on. - - `db`: - Default named database to operate on. If unspecified, defaults to - the environment's main database. Can be overridden on a per-call - basis below. - - `parent`: - ``None``, or a parent transaction (see lmdb.h). - - `write`: - Transactions are read-only by default. To modify the database, you - must pass `write=True`. This flag is ignored if - :py:class:`Environment` was opened with ``readonly=True``. - - `buffers`: - If ``True``, indicates :py:func:`buffer` objects should be yielded - instead of bytestrings. This setting applies to the - :py:class:`Transaction` instance itself and any :py:class:`Cursors - ` created within the transaction. - - This feature significantly improves performance, since MDB has a - zero-copy design, but it requires care when manipulating the - returned buffer objects. The benefit of this facility is diminished - when using small keys and values. - """ - - # If constructor fails, then __del__ will attempt to access these - # attributes. - _env = _invalid - _txn = _invalid - _parent = None - _write = False - - # Mutations occurred since transaction start. Required to know when Cursor - # key/value must be refreshed. - _mutations = 0 - - def __init__(self, env, db=None, parent=None, write=False, buffers=False): - env._deps.add(self) - self.env = env # hold ref - self._db = db or env._db - self._env = env._env - self._key = _ffi.new('MDB_val *') - self._val = _ffi.new('MDB_val *') - self._to_py = _mvbuf if buffers else _mvstr - self._deps = set() - - if parent: - self._parent = parent - parent_txn = parent._txn - parent._deps.add(self) - else: - parent_txn = _ffi.NULL - - if write: - if env.readonly: - msg = 'Cannot start write transaction with read-only env' - raise _error(msg, _lib.EACCES) - - txnpp = _ffi.new('MDB_txn **') - rc = _lib.mdb_txn_begin(self._env, parent_txn, 0, txnpp) - if rc: - raise _error("mdb_txn_begin", rc) - self._txn = txnpp[0] - self._write = True - else: - try: # Exception catch in order to avoid racy 'if txns:' test - self._txn = env._spare_txns.pop() - env._max_spare_txns += 1 - rc = _lib.mdb_txn_renew(self._txn) - if rc: - _lib.mdb_txn_abort(self._txn) - raise _error("mdb_txn_renew", rc) - except IndexError: - txnpp = _ffi.new('MDB_txn **') - flags = _lib.MDB_RDONLY - rc = _lib.mdb_txn_begin(self._env, parent_txn, flags, txnpp) - if rc: - raise _error("mdb_txn_begin", rc) - self._txn = txnpp[0] - - def _invalidate(self): - if self._txn: - self.abort() - self.env._deps.discard(self) - self._parent = None - self._env = _invalid - - def __del__(self): - self.abort() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - if exc_type: - self.abort() - else: - self.commit() - - def stat(self, db): - """stat(db) - - Return statistics like :py:meth:`Environment.stat`, except for a single - DBI. `db` must be a database handle returned by :py:meth:`open_db`. - """ - st = _ffi.new('MDB_stat *') - rc = _lib.mdb_stat(self._txn, db._dbi, st) - if rc: - raise _error('mdb_stat', rc) - return self.env._convert_stat(st) - - def drop(self, db, delete=True): - """Delete all keys in a named database and optionally delete the named - database itself. Deleting the named database causes it to become - unavailable, and invalidates existing cursors. - - Equivalent to `mdb_drop() - `_ - """ - while db._deps: - db._deps.pop()._invalidate() - rc = _lib.mdb_drop(self._txn, db._dbi, delete) - self._mutations += 1 - if rc: - raise _error("mdb_drop", rc) - - def _cache_spare(self): - # In order to avoid taking and maintaining a lock, a race is allowed - # below which may result in more spare txns than desired. It seems - # unlikely the race could ever result in a large amount of spare txns, - # and in any case a correctly configured program should not be opening - # more read-only transactions than there are configured spares. - if self.env._max_spare_txns > 0: - _lib.mdb_txn_reset(self._txn) - self.env._spare_txns.append(self._txn) - self.env._max_spare_txns -= 1 - self._txn = _invalid - self._invalidate() - return True - - def commit(self): - """Commit the pending transaction. - - Equivalent to `mdb_txn_commit() - `_ - """ - while self._deps: - self._deps.pop()._invalidate() - if self._write or not self._cache_spare(): - rc = _lib.mdb_txn_commit(self._txn) - self._txn = _invalid - if rc: - raise _error("mdb_txn_commit", rc) - self._invalidate() - - def abort(self): - """Abort the pending transaction. Repeat calls to :py:meth:`abort` have - no effect after a previously successful :py:meth:`commit` or - :py:meth:`abort`, or after the associated :py:class:`Environment` has - been closed. - - Equivalent to `mdb_txn_abort() - `_ - """ - if self._txn: - while self._deps: - self._deps.pop()._invalidate() - if self._write or not self._cache_spare(): - rc = _lib.mdb_txn_abort(self._txn) - self._txn = _invalid - if rc: - raise _error("mdb_txn_abort", rc) - self._invalidate() - - def get(self, key, default=None, db=None): - """Fetch the first value matching `key`, returning `default` if `key` - does not exist. A cursor must be used to fetch all values for a key in - a `dupsort=True` database. - - Equivalent to `mdb_get() - `_ - """ - rc = _lib.pymdb_get(self._txn, (db or self._db)._dbi, - key, len(key), self._val) - if rc: - if rc == _lib.MDB_NOTFOUND: - return default - raise _error("mdb_cursor_get", rc) - return self._to_py(self._val) - - def put(self, key, value, dupdata=True, overwrite=True, append=False, - db=None): - """Store a record, returning ``True`` if it was written, or ``False`` - to indicate the key was already present and `overwrite=False`. - - Equivalent to `mdb_put() - `_ - - `key`: - Bytestring key to store. - - `value`: - Bytestring value to store. - - `dupdata`: - If ``True`` and database was opened with `dupsort=True`, add - pair as a duplicate if the given key already exists. Otherwise - overwrite any existing matching key. - - `overwrite`: - If ``False``, do not overwrite any existing matching key. - - `append`: - If ``True``, append the pair to the end of the database without - comparing its order first. Appending a key that is not greater - than the highest existing key will cause corruption. - - `db`: - Named database to operate on. If unspecified, defaults to the - database given to the :py:class:`Transaction` constructor. - """ - flags = 0 - if not dupdata: - flags |= _lib.MDB_NODUPDATA - if not overwrite: - flags |= _lib.MDB_NOOVERWRITE - if append: - flags |= _lib.MDB_APPEND - - rc = _lib.pymdb_put(self._txn, (db or self._db)._dbi, - key, len(key), value, len(value), flags) - self._mutations += 1 - if rc: - if rc == _lib.MDB_KEYEXIST: - return False - raise _error("mdb_put", rc) - return True - - def replace(self, key, value, db=None): - """Use a temporary cursor to invoke :py:meth:`Cursor.replace`. - - `db`: - Named database to operate on. If unspecified, defaults to the - database given to the :py:class:`Transaction` constructor. - """ - with Cursor(db or self._db, self) as curs: - return curs.replace(key, value) - - def pop(self, key, db=None): - """Use a temporary cursor to invoke :py:meth:`Cursor.pop`. - - `db`: - Named database to operate on. If unspecified, defaults to the - database given to the :py:class:`Transaction` constructor. - """ - with Cursor(db or self._db, self) as curs: - return curs.pop(key) - - def delete(self, key, value=EMPTY_BYTES, db=None): - """Delete a key from the database. - - Equivalent to `mdb_del() - `_ - - `key`: - The key to delete. - - value: - If the database was opened with dupsort=True and value is not - the empty bytestring, then delete elements matching only this - `(key, value)` pair, otherwise all values for key are deleted. - - Returns True if at least one key was deleted. - """ - rc = _lib.pymdb_del(self._txn, (db or self._db)._dbi, - key, len(key), value, len(value)) - self._mutations += 1 - if rc: - if rc == _lib.MDB_NOTFOUND: - return False - raise _error("mdb_del", rc) - return True - - def cursor(self, db=None): - """Shortcut for ``lmdb.Cursor(db, self)``""" - return Cursor(db or self._db, self) - - -class Cursor(object): - """ - Structure for navigating a database. - - Equivalent to `mdb_cursor_open() - `_ - - `db`: - :py:class:`Database` to navigate. - - `txn`: - :py:class:`Transaction` to navigate. - - As a convenience, :py:meth:`Transaction.cursor` can be used to quickly - return a cursor: - - :: - - >>> env = lmdb.open('/tmp/foo') - >>> child_db = env.open_db('child_db') - >>> with env.begin() as txn: - ... cursor = txn.cursor() # Cursor on main database. - ... cursor2 = txn.cursor(child_db) # Cursor on child database. - - Cursors start in an unpositioned state. If :py:meth:`iternext` or - :py:meth:`iterprev` are used in this state, iteration proceeds from the - start or end respectively. Iterators directly position using the cursor, - meaning strange behavior results when multiple iterators exist on the same - cursor. - - .. note:: - - From the perspective of the Python binding, cursors return to an - 'unpositioned' state once any scanning or seeking method (e.g. - :py:meth:`next`, :py:meth:`prev_nodup`, :py:meth:`set_range`) returns - ``False`` or raises an exception. This is primarily to ensure safe, - consistent semantics in the face of any error condition. - - When the Cursor returns to an unpositioned state, its :py:meth:`key` - and :py:meth:`value` return empty strings to indicate there is no - active position, although internally the LMDB cursor may still have a - valid position. - - This may lead to slightly surprising behaviour when iterating the - values for a `dupsort=True` database's keys, since methods such as - :py:meth:`iternext_dup` will cause Cursor to appear unpositioned, - despite it returning ``False`` only to indicate there are no more - values for the current key. In that case, simply calling - :py:meth:`next` would cause iteration to resume at the next available - key. - - This behaviour may change in future. - - Iterator methods such as :py:meth:`iternext` and :py:meth:`iterprev` accept - `keys` and `values` arguments. If both are ``True``, then the value of - :py:meth:`item` is yielded on each iteration. If only `keys` is ``True``, - :py:meth:`key` is yielded, otherwise only :py:meth:`value` is yielded. - - Prior to iteration, a cursor can be positioned anywhere in the database: - - :: - - >>> with env.begin() as txn: - ... cursor = txn.cursor() - ... if not cursor.set_range('5'): # Position at first key >= '5'. - ... print('Not found!') - ... else: - ... for key, value in cursor: # Iterate from first key >= '5'. - ... print((key, value)) - - Iteration is not required to navigate, and sometimes results in ugly or - inefficient code. In cases where the iteration order is not obvious, or is - related to the data being read, use of :py:meth:`set_key`, - :py:meth:`set_range`, :py:meth:`key`, :py:meth:`value`, and :py:meth:`item` - may be preferable: - - :: - - >>> # Record the path from a child to the root of a tree. - >>> path = ['child14123'] - >>> while path[-1] != 'root': - ... assert cursor.set_key(path[-1]), \\ - ... 'Tree is broken! Path: %s' % (path,) - ... path.append(cursor.value()) - """ - def __init__(self, db, txn): - db._deps.add(self) - txn._deps.add(self) - self.db = db # hold ref - self.txn = txn # hold ref - self._dbi = db._dbi - self._txn = txn._txn - self._key = _ffi.new('MDB_val *') - self._val = _ffi.new('MDB_val *') - self._valid = False - self._to_py = txn._to_py - curpp = _ffi.new('MDB_cursor **') - self._cur = None - rc = _lib.mdb_cursor_open(self._txn, self._dbi, curpp) - if rc: - raise _error("mdb_cursor_open", rc) - self._cur = curpp[0] - # If Transaction.mutations!=last_mutation, must MDB_GET_CURRENT to - # refresh `key' and `val'. - self._last_mutation = txn._mutations - - def _invalidate(self): - if self._cur: - _lib.mdb_cursor_close(self._cur) - self.db._deps.discard(self) - self.txn._deps.discard(self) - self._cur = _invalid - self._dbi = _invalid - self._txn = _invalid - - def __del__(self): - self._invalidate() - - def close(self): - """Close the cursor, freeing its associated resources.""" - self._invalidate() - - def __enter__(self): - return self - - def __exit__(self, _1, _2, _3): - self._invalidate() - - def key(self): - """Return the current key.""" - # Must refresh `key` and `val` following mutation. - if self._last_mutation != self.txn._mutations: - self._cursor_get(_lib.MDB_GET_CURRENT) - return self._to_py(self._key) - - def value(self): - """Return the current value.""" - # Must refresh `key` and `val` following mutation. - if self._last_mutation != self.txn._mutations: - self._cursor_get(_lib.MDB_GET_CURRENT) - return self._to_py(self._val) - - def item(self): - """Return the current `(key, value)` pair.""" - # Must refresh `key` and `val` following mutation. - if self._last_mutation != self.txn._mutations: - self._cursor_get(_lib.MDB_GET_CURRENT) - return self._to_py(self._key), self._to_py(self._val) - - def _iter(self, op, keys, values): - if not values: - get = self.key - elif not keys: - get = self.value - else: - get = self.item - - cur = self._cur - key = self._key - val = self._val - while self._valid: - yield get() - rc = _lib.mdb_cursor_get(cur, key, val, op) - self._valid = not rc - if rc and rc != _lib.MDB_NOTFOUND: - raise _error("mdb_cursor_get", rc) - - def iternext(self, keys=True, values=True): - """Return a forward iterator that yields the current element before - calling :py:meth:`next`, repeating until the end of the database is - reached. As a convenience, :py:class:`Cursor` implements the iterator - protocol by automatically returning a forward iterator when invoked: - - :: - - >>> # Equivalent: - >>> it = iter(cursor) - >>> it = cursor.iternext(keys=True, values=True) - - If the cursor is not yet positioned, it is moved to the first key in - the database, otherwise iteration proceeds from the current position. - """ - if not self._valid: - self.first() - return self._iter(_lib.MDB_NEXT, keys, values) - __iter__ = iternext - - def iternext_dup(self, keys=False, values=True): - """Return a forward iterator that yields the current value - ("duplicate") of the current key before calling :py:meth:`next_dup`, - repeating until the last value of the current key is reached. - - Only meaningful for databases opened with `dupsort=True`. - - .. code-block:: python - - if not cursor.set_key("foo"): - print("No values found for 'foo'") - else: - for idx, data in enumerate(cursor.iternext_dup()): - print("%d'th value for 'foo': %s" % (idx, data)) - """ - return self._iter(_lib.MDB_NEXT_DUP, keys, values) - - def iternext_nodup(self, keys=True, values=False): - """Return a forward iterator that yields the current value - ("duplicate") of the current key before calling :py:meth:`next_nodup`, - repeating until the end of the database is reached. - - Only meaningful for databases opened with `dupsort=True`. - - If the cursor is not yet positioned, it is moved to the first key in - the database, otherwise iteration proceeds from the current position. - - .. code-block:: python - - for key in cursor.iternext_nodup(): - print("Key '%s' has %d values" % (key, cursor.count())) - """ - if not self._valid: - self.first() - return self._iter(_lib.MDB_NEXT_NODUP, keys, values) - - def iterprev(self, keys=True, values=True): - """Return a reverse iterator that yields the current element before - calling :py:meth:`prev`, until the start of the database is reached. - - If the cursor is not yet positioned, it is moved to the last key in - the database, otherwise iteration proceeds from the current position. - - :: - - >>> with env.begin() as txn: - ... for i, (key, value) in enumerate(txn.cursor().iterprev()): - ... print('%dth last item is (%r, %r)' % (1+i, key, value)) - """ - if not self._valid: - self.last() - return self._iter(_lib.MDB_PREV, keys, values) - - def iterprev_dup(self, keys=False, values=True): - """Return a reverse iterator that yields the current value - ("duplicate") of the current key before calling :py:meth:`prev_dup`, - repeating until the first value of the current key is reached. - - Only meaningful for databases opened with `dupsort=True`. - """ - return self._iter(_lib.MDB_PREV_DUP, keys, values) - - def iterprev_nodup(self, keys=True, values=False): - """Return a reverse iterator that yields the current value - ("duplicate") of the current key before calling :py:meth:`prev_nodup`, - repeating until the start of the database is reached. - - If the cursor is not yet positioned, it is moved to the last key in - the database, otherwise iteration proceeds from the current position. - - Only meaningful for databases opened with `dupsort=True`. - """ - if not self._valid: - self.last() - return self._iter(_lib.MDB_PREV_NODUP, keys, values) - - def _cursor_get(self, op): - rc = _lib.mdb_cursor_get(self._cur, self._key, self._val, op) - self._valid = v = not rc - self._last_mutation = self.txn._mutations - if rc: - self._key.mv_size = 0 - self._val.mv_size = 0 - if rc != _lib.MDB_NOTFOUND: - if not (rc == _lib.EINVAL and op == _lib.MDB_GET_CURRENT): - raise _error("mdb_cursor_get", rc) - return v - - def _cursor_get_kv(self, op, k, v): - rc = _lib.pymdb_cursor_get(self._cur, k, len(k), v, len(v), - self._key, self._val, op) - self._valid = v = not rc - if rc: - self._key.mv_size = 0 - self._val.mv_size = 0 - if rc != _lib.MDB_NOTFOUND: - if not (rc == _lib.EINVAL and op == _lib.MDB_GET_CURRENT): - raise _error("mdb_cursor_get", rc) - return v - - def first(self): - """Move to the first key in the database, returning ``True`` on success - or ``False`` if the database is empty. - - If the database was opened with `dupsort=True` and the key contains - duplicates, the cursor is positioned on the first value ("duplicate"). - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_FIRST - `_ - """ - return self._cursor_get(_lib.MDB_FIRST) - - def first_dup(self): - """Move to the first value ("duplicate") for the current key, returning - ``True`` on success or ``False`` if the database is empty. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_FIRST_DUP - `_ - """ - return self._cursor_get(_lib.MDB_FIRST_DUP) - - def last(self): - """Move to the last key in the database, returning ``True`` on success - or ``False`` if the database is empty. - - If the database was opened with `dupsort=True` and the key contains - duplicates, the cursor is positioned on the last value ("duplicate"). - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_LAST - `_ - """ - return self._cursor_get(_lib.MDB_LAST) - - def last_dup(self): - """Move to the last value ("duplicate") for the current key, returning - ``True`` on success or ``False`` if the database is empty. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_LAST_DUP - `_ - """ - return self._cursor_get(_lib.MDB_LAST_DUP) - - def prev(self): - """Move to the previous element, returning ``True`` on success or - ``False`` if there is no previous item. - - For databases opened with `dupsort=True`, moves to the previous data - item ("duplicate") for the current key if one exists, otherwise moves - to the previous key. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_PREV - `_ - """ - return self._cursor_get(_lib.MDB_PREV) - - def prev_dup(self): - """Move to the previous value ("duplicate") of the current key, - returning ``True`` on success or ``False`` if there is no previous - value. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_PREV_DUP - `_ - """ - return self._cursor_get(_lib.MDB_PREV_DUP) - - def prev_nodup(self): - """Move to the last value ("duplicate") of the previous key, returning - ``True`` on success or ``False`` if there is no previous key. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_PREV_NODUP - `_ - """ - return self._cursor_get(_lib.MDB_PREV_NODUP) - - def next(self): - """Move to the next element, returning ``True`` on success or ``False`` - if there is no next element. - - For databases opened with `dupsort=True`, moves to the next value - ("duplicate") for the current key if one exists, otherwise moves to the - first value of the next key. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_NEXT - `_ - """ - return self._cursor_get(_lib.MDB_NEXT) - - def next_dup(self): - """Move to the next value ("duplicate") of the current key, returning - ``True`` on success or ``False`` if there is no next value. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_NEXT_DUP - `_ - """ - return self._cursor_get(_lib.MDB_NEXT_DUP) - - def next_nodup(self): - """Move to the first value ("duplicate") of the next key, returning - ``True`` on success or ``False`` if there is no next key. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_NEXT_NODUP - `_ - """ - return self._cursor_get(_lib.MDB_PREV_NODUP) - - def set_key(self, key): - """Seek exactly to `key`, returning ``True`` on success or ``False`` if - the exact key was not found. It is an error to :py:meth:`set_key` the - empty bytestring. - - For databases opened with `dupsort=True`, moves to the first value - ("duplicate") for the key. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_SET_KEY - `_ - """ - return self._cursor_get_kv(_lib.MDB_SET_KEY, key, EMPTY_BYTES) - - def set_key_dup(self, key, value): - """Seek exactly to `(key, value)`, returning ``True`` on success or - ``False`` if the exact key and value was not found. It is an error - to :py:meth:`set_key` the empty bytestring. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_GET_BOTH - `_ - """ - return self._cursor_get_kv(_lib.MDB_GET_BOTH, key, value) - - def get(self, key, default=None): - """Equivalent to :py:meth:`set_key()`, except :py:meth:`value` is - returned when `key` is found, otherwise `default`. - """ - if self._cursor_get_kv(_lib.MDB_SET_KEY, key, EMPTY_BYTES): - return self.value() - return default - - def set_range(self, key): - """Seek to the first key greater than or equal to `key`, returning - ``True`` on success, or ``False`` to indicate key was past end of - database. Behaves like :py:meth:`first` if `key` is the empty - bytestring. - - For databases opened with `dupsort=True`, moves to the first value - ("duplicate") for the key. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_SET_RANGE - `_ - """ - if not key: - return self.first() - return self._cursor_get_kv(_lib.MDB_SET_RANGE, key, EMPTY_BYTES) - - def set_range_dup(self, key, value): - """Seek to the first key/value pair greater than or equal to `key`, - returning ``True`` on success, or ``False`` to indicate `(key, value)` - was past end of database. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_get() - `_ - with `MDB_GET_BOTH_RANGE - `_ - """ - return self._cursor_get_kv(_lib.MDB_GET_BOTH_RANGE, key, value) - - def delete(self, dupdata=False): - """Delete the current element and move to the next, returning ``True`` - on success or ``False`` if the database was empty. - - If `dupdata` is ``True``, delete all values ("duplicates") for the - current key, otherwise delete only the currently positioned value. Only - meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_del() - `_ - """ - v = self._valid - if v: - flags = _lib.MDB_NODUPDATA if dupdata else 0 - rc = _lib.mdb_cursor_del(self._cur, flags) - self.txn._mutations += 1 - if rc: - raise _error("mdb_cursor_del", rc) - self._cursor_get(_lib.MDB_GET_CURRENT) - v = rc == 0 - return v - - def count(self): - """Return the number of values ("duplicates") for the current key. - - Only meaningful for databases opened with `dupsort=True`. - - Equivalent to `mdb_cursor_count() - `_ - """ - countp = _ffi.new('size_t *') - rc = _lib.mdb_cursor_count(self._cur, countp) - if rc: - raise _error("mdb_cursor_count", rc) - return countp[0] - - def put(self, key, val, dupdata=True, overwrite=True, append=False): - """Store a record, returning ``True`` if it was written, or ``False`` - to indicate the key was already present and `overwrite=False`. On - success, the cursor is positioned on the key. - - Equivalent to `mdb_cursor_put() - `_ - - `key`: - Bytestring key to store. - - `val`: - Bytestring value to store. - - `dupdata`: - If ``True`` and database was opened with `dupsort=True`, add - pair as a duplicate if the given key already exists. Otherwise - overwrite any existing matching key. - - `overwrite`: - If ``False``, do not overwrite the value for the key if it - exists, just return ``False``. For databases opened with - `dupsort=True`, ``False`` will always be returned if a - duplicate key/value pair is inserted, regardless of the setting - for `overwrite`. - - `append`: - If ``True``, append the pair to the end of the database without - comparing its order first. Appending a key that is not greater - than the highest existing key will cause corruption. - """ - flags = 0 - if not dupdata: - flags |= _lib.MDB_NODUPDATA - if not overwrite: - flags |= _lib.MDB_NOOVERWRITE - if append: - flags |= _lib.MDB_APPEND - - rc = _lib.pymdb_cursor_put(self._cur, key, len(key), val, len(val), flags) - self.txn._mutations += 1 - if rc: - if rc == _lib.MDB_KEYEXIST: - return False - raise _error("mdb_cursor_put", rc) - self._cursor_get(_lib.MDB_GET_CURRENT) - return True - - def putmulti(self, items, dupdata=True, overwrite=True, append=False): - """Invoke :py:meth:`put` for each `(key, value)` 2-tuple from the - iterable `items`. Elements must be exactly 2-tuples, they may not be of - any other type, or tuple subclass. - - Returns a tuple `(consumed, added)`, where `consumed` is the number of - elements read from the iterable, and `added` is the number of new - entries added to the database. `added` may be less than `consumed` when - `overwrite=False`. - - `items`: - Iterable to read records from. - - `dupdata`: - If ``True`` and database was opened with `dupsort=True`, add - pair as a duplicate if the given key already exists. Otherwise - overwrite any existing matching key. - - `overwrite`: - If ``False``, do not overwrite the value for the key if it - exists, just return ``False``. For databases opened with - `dupsort=True`, ``False`` will always be returned if a - duplicate key/value pair is inserted, regardless of the setting - for `overwrite`. - - `append`: - If ``True``, append records to the end of the database without - comparing their order first. Appending a key that is not - greater than the highest existing key will cause corruption. - """ - flags = 0 - if not dupdata: - flags |= _lib.MDB_NODUPDATA - if not overwrite: - flags |= _lib.MDB_NOOVERWRITE - if append: - flags |= _lib.MDB_APPEND - - added = 0 - skipped = 0 - for key, value in items: - rc = _lib.pymdb_cursor_put(self._cur, key, len(key), - value, len(value), flags) - self.txn._mutations += 1 - added += 1 - if rc: - if rc == _lib.MDB_KEYEXIST: - skipped += 1 - else: - raise _error("mdb_cursor_put", rc) - self._cursor_get(_lib.MDB_GET_CURRENT) - return added, added-skipped - - def replace(self, key, val): - """Store a record, returning its previous value if one existed. Returns - ``None`` if no previous value existed. This uses the best available - mechanism to minimize the cost of a `set-and-return-previous` - operation. - - For databases opened with `dupsort=True`, only the first data element - ("duplicate") is returned if it existed, all data elements are removed - and the new `(key, data)` pair is inserted. - - `key`: - Bytestring key to store. - - `value`: - Bytestring value to store. - """ - if self.db._flags & _lib.MDB_DUPSORT: - if self._cursor_get_kv(_lib.MDB_SET_KEY, key, EMPTY_BYTES): - old = _mvstr(self._val) - self.delete(True) - else: - old = None - self.put(key, val) - return old - - flags = _lib.MDB_NOOVERWRITE - keylen = len(key) - rc = _lib.pymdb_cursor_put(self._cur, key, keylen, val, len(val), flags) - self.txn._mutations += 1 - if not rc: - return - if rc != _lib.MDB_KEYEXIST: - raise _error("mdb_cursor_put", rc) - - self._cursor_get(_lib.MDB_GET_CURRENT) - old = _mvstr(self._val) - rc = _lib.pymdb_cursor_put(self._cur, key, keylen, val, len(val), 0) - self.txn._mutations += 1 - if rc: - raise _error("mdb_cursor_put", rc) - self._cursor_get(_lib.MDB_GET_CURRENT) - return old - - def pop(self, key): - """Fetch a record's value then delete it. Returns ``None`` if no - previous value existed. This uses the best available mechanism to - minimize the cost of a `delete-and-return-previous` operation. - - For databases opened with `dupsort=True`, the first data element - ("duplicate") for the key will be popped. - - `key`: - Bytestring key to delete. - """ - if self._cursor_get_kv(_lib.MDB_SET_KEY, key, EMPTY_BYTES): - old = _mvstr(self._val) - rc = _lib.mdb_cursor_del(self._cur, 0) - self.txn._mutations += 1 - if rc: - raise _error("mdb_cursor_del", rc) - self._cursor_get(_lib.MDB_GET_CURRENT) - return old - - def _iter_from(self, k, reverse): - """Helper for centidb. Please do not rely on this interface, it may be - removed in future. - """ - if not k and not reverse: - found = self.first() - else: - found = self.set_range(k) - if reverse: - if not found: - self.last() - return self.iterprev() - else: - if not found: - return iter(()) - return self.iternext() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/cpython.c stb-tester-31/vendor/py-lmdb/lmdb/cpython.c --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/cpython.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lmdb/cpython.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,3637 +0,0 @@ -/* - * Copyright 2013 The py-lmdb authors, all rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * OpenLDAP is a registered trademark of the OpenLDAP Foundation. - * - * Individual files and/or contributed packages may be copyright by - * other parties and/or subject to additional restrictions. - * - * This work also contains materials derived from public sources. - * - * Additional information about OpenLDAP can be obtained at - * . - */ - -#define PY_SSIZE_T_CLEAN - -/* Search lib/win32 first, then fallthrough to as required.*/ -#include "stdint.h" - -#include -#include -#include -#include - -#include "Python.h" -#include "structmember.h" - -#ifdef HAVE_MEMSINK -#define USING_MEMSINK -#include "memsink.h" -#endif - -#ifdef _WIN32 -#include /* HANDLE */ -#endif - -#include "lmdb.h" -#include "preload.h" - - -/* Comment out for copious debug. */ -#define NODEBUG - -#ifdef NODEBUG -# define DEBUG(s, ...) -#else -# define DEBUG(s, ...) fprintf(stderr, \ - "lmdb.cpython: %s:%d: " s "\n", __func__, __LINE__, ## __VA_ARGS__); -#endif - -#define MDEBUG(s, ...) DEBUG("%p: " s, self, ## __VA_ARGS__); - - -/* Inlining control for compatible compilers. */ -#if (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define NOINLINE __attribute__((noinline)) -#else -# define NOINLINE -#endif - - -/** - * On Win32, Environment.copyfd() needs _get_osfmodule() from the C library, - * except that function performs no input validation. So instead we import - * msvcrt standard library module, which wraps _get_osfmodule() in a way that - * is crash-safe. - */ -#ifdef _WIN32 -static PyObject *msvcrt; -#endif - - -/** PyLong representing integer 0. */ -static PyObject *py_zero; -/** PyLong representing INT_MAX. */ -static PyObject *py_int_max; -/** PyLong representing SIZE_MAX. */ -static PyObject *py_size_max; -/** lmdb.Error type. */ -static PyObject *Error; - -/** Typedefs and forward declarations. */ -static PyTypeObject PyDatabase_Type; -static PyTypeObject PyEnvironment_Type; -static PyTypeObject PyTransaction_Type; -static PyTypeObject PyCursor_Type; -static PyTypeObject PyIterator_Type; - -typedef struct CursorObject CursorObject; -typedef struct DbObject DbObject; -typedef struct EnvObject EnvObject; -typedef struct IterObject IterObject; -typedef struct TransObject TransObject; - - -/* ------------------------ */ -/* Python 3.x Compatibility */ -/* ------------------------ */ - -#if PY_MAJOR_VERSION >= 3 - -# define MOD_RETURN(mod) return mod; -# define MODINIT_NAME PyInit_cpython - -# define MAKE_ID(id) PyCapsule_New((void *) (1 + (id)), NULL, NULL) -# define READ_ID(obj) (((int) (long) PyCapsule_GetPointer(obj, NULL)) - 1) - -#else - -# define MOD_RETURN(mod) return -# define MODINIT_NAME initcpython - -# define MAKE_ID(id) PyInt_FromLong((long) id) -# define READ_ID(obj) PyInt_AS_LONG(obj) - -# define PyUnicode_InternFromString PyString_InternFromString -# define PyBytes_AS_STRING PyString_AS_STRING -# define PyBytes_GET_SIZE PyString_GET_SIZE -# define PyBytes_CheckExact PyString_CheckExact -# define PyBytes_FromStringAndSize PyString_FromStringAndSize -# define _PyBytes_Resize _PyString_Resize -# define PyMemoryView_FromMemory(x, y, z) PyBuffer_FromMemory(x, y) - -# ifndef PyBUF_READ -# define PyBUF_READ 0 -# endif - -/* Python 2.5 */ -# ifndef Py_TYPE -# define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -# endif - -# ifndef PyVarObject_HEAD_INIT -# define PyVarObject_HEAD_INIT(x, y) \ - PyObject_HEAD_INIT(x) y, -# endif - -#endif - -#if (PY_MAJOR_VERSION == 2) && (PY_MINOR_VERSION < 6) -static PyObject * -PyUnicode_FromString(const char *u) -{ - PyObject *s = PyString_FromString(u); - if(s) { - PyObject *t = PyUnicode_FromEncodedObject( - s, Py_FileSystemDefaultEncoding, "strict"); - Py_DECREF(s); - s = t; - } - return s; -} -#endif - - -struct list_head { - struct lmdb_object *prev; - struct lmdb_object *next; -}; - -#define LmdbObject_HEAD \ - PyObject_HEAD \ - struct list_head siblings; \ - struct list_head children; \ - int valid; - -struct lmdb_object { - LmdbObject_HEAD -}; - -#define OBJECT_INIT(o) \ - ((struct lmdb_object *)o)->siblings.prev = NULL; \ - ((struct lmdb_object *)o)->siblings.next = NULL; \ - ((struct lmdb_object *)o)->children.prev = NULL; \ - ((struct lmdb_object *)o)->children.next = NULL; \ - ((struct lmdb_object *)o)->valid = 1; - - -/** lmdb._Database */ -struct DbObject { - LmdbObject_HEAD - /** Python Environment reference. Not refcounted; when the last strong ref - * to Environment is released, DbObject.tp_clear() will be called, causing - * DbObject.env and DbObject.dbi to be cleared. This is to prevent a - * cyclical reference from DB->Env keeping the environment alive. */ - struct EnvObject *env; - /** MDB database handle. */ - MDB_dbi dbi; - /** Flags at time of creation. */ - unsigned int flags; -}; - -/** lmdb.Environment */ -struct EnvObject { - LmdbObject_HEAD - /** Python-managed list of weakrefs to this object. */ - PyObject *weaklist; - /** MDB environment object. */ - MDB_env *env; - /** DBI for main database, opened during Environment construction. */ - DbObject *main_db; - /** 1 if env opened read-only; transactions must always be read-only. */ - int readonly; - - /** Max free txns to keep on free_txn list. */ - int max_spare_txns; - /** Spare read-only transaction list head. */ - struct TransObject *spare_txns; -}; - -/** TransObject.flags bitfield values. */ -enum trans_flags { - /** Buffers should be yielded by get. */ - TRANS_BUFFERS = 1, - /** Transaction can be can go on freelist instead of deallocation. */ - TRANS_RDONLY = 2, - /** Transaction is spare, ready for mdb_txn_renew() */ - TRANS_SPARE = 4 -}; - -/** lmdb.Transaction */ -struct TransObject { - LmdbObject_HEAD - /** Python-managed list of weakrefs to this object. */ - PyObject *weaklist; - EnvObject *env; -#ifdef HAVE_MEMSINK - /** Copy-on-invalid list head. */ - PyObject *sink_head; -#endif - /** MDB transaction object. */ - MDB_txn *txn; - /** Bitfield of trans_flags values. */ - int flags; - /** Default database if none specified. */ - DbObject *db; - /** Number of mutations occurred since start of transaction. Required to - * know when cursor key/value must be refreshed. */ - int mutations; - /** Next free read-only txn, or NULL. */ - struct TransObject *spare_next; -}; - -/** lmdb.Cursor */ -struct CursorObject { - LmdbObject_HEAD - /** Transaction cursor belongs to. */ - TransObject *trans; - /** 1 if mdb_cursor_get() has been called and it last returned 0. */ - int positioned; - /** MDB-level cursor object. */ - MDB_cursor *curs; - /** mv_size==0 if positioned==0, otherwise points to current key. */ - MDB_val key; - /** mv_size==0 if positioned==0, otherwise points to current value. */ - MDB_val val; - /** If TransObject.mutations!=last_mutation, must MDB_GET_CURRENT to - * refresh `key' and `val'. */ - int last_mutation; - /** DBI flags at time of creation. */ - unsigned int dbi_flags; -}; - - -typedef PyObject *(*IterValFunc)(CursorObject *); - -/** lmdb.Iterator - * - * This is separate from Cursor since we want to define Cursor.next() to mean - * MDB_NEXT, and a Python iterator's next() has different semantics. - */ -struct IterObject { - PyObject_HEAD - /** Cursor being iterated, or NULL for freelist iterator. */ - CursorObject *curs; - /** 1 if iteration has started (Cursor should advance on next()). */ - int started; - /** Operation used to advance cursor. */ - MDB_cursor_op op; - /** Iterator value function, should be item(), key(), or value(). */ - IterValFunc val_func; -}; - - -/** - * Link `child` into `parent`'s list of dependent objects. Use LINK_CHILD() - * maro to avoid casting PyObject to lmdb_object. - */ -static void link_child(struct lmdb_object *parent, struct lmdb_object *child) -{ - struct lmdb_object *sibling = parent->children.next; - if(sibling) { - child->siblings.next = sibling; - sibling->siblings.prev = child; - } - parent->children.next = child; -} - -#define LINK_CHILD(parent, child) link_child((void *)parent, (void *)child); - - -/** - * Remove `child` from `parent`'s list of dependent objects. Use UNLINK_CHILD - * macro to avoid casting PyObject to lmdb_object. - */ -static void unlink_child(struct lmdb_object *parent, struct lmdb_object *child) -{ - if(parent) { - struct lmdb_object *prev = child->siblings.prev; - struct lmdb_object *next = child->siblings.next; - if(prev) { - prev->siblings.next = next; - /* If double unlink_child(), this test my legitimately fail: */ - } else if(parent->children.next == child) { - parent->children.next = next; - } - if(next) { - next->siblings.prev = prev; - } - child->siblings.prev = NULL; - child->siblings.next = NULL; - } -} - -#define UNLINK_CHILD(parent, child) unlink_child((void *)parent, (void *)child); - - -/** - * Notify dependents of `parent` that `parent` is about to become invalid, - * and that they should free any dependent resources. - * - * To save effort, tp_clear is overloaded to be the invalidation function, - * instead of carrying a separate pointer. Objects are added to their parent's - * list during construction and removed during deallocation. - * - * When the environment is closed, it walks its list calling tp_clear on each - * child, which in turn walk their own lists. Child transactions are added to - * their parent transaction's list. Iterators keep no significant state, so - * they are not tracked. - * - * Use INVALIDATE() macro to avoid casting PyObject to lmdb_object. - */ -static void invalidate(struct lmdb_object *parent) -{ - struct lmdb_object *child = parent->children.next; - while(child) { - struct lmdb_object *next = child->siblings.next; - DEBUG("invalidating parent=%p child %p", parent, child) - Py_TYPE(child)->tp_clear((PyObject *) child); - child = next; - } -} - -#define INVALIDATE(parent) invalidate((void *)parent); - - -/* ---------- */ -/* Exceptions */ -/* ---------- */ - -struct error_map { - int code; - const char *name; -}; - -/** Array of Error subclasses corresponding to `error_map'. */ -static PyObject **error_tbl; -/** Mapping from LMDB error code to py-lmdb exception class. */ -static const struct error_map error_map[] = { - {MDB_KEYEXIST, "KeyExistsError"}, - {MDB_NOTFOUND, "NotFoundError"}, - {MDB_PAGE_NOTFOUND, "PageNotFoundError"}, - {MDB_CORRUPTED, "CorruptedError"}, - {MDB_PANIC, "PanicError"}, - {MDB_VERSION_MISMATCH, "VersionMismatchError"}, - {MDB_INVALID, "InvalidError"}, - {MDB_MAP_FULL, "MapFullError"}, - {MDB_DBS_FULL, "DbsFullError"}, - {MDB_READERS_FULL, "ReadersFullError"}, - {MDB_TLS_FULL, "TlsFullError"}, - {MDB_TXN_FULL, "TxnFullError"}, - {MDB_CURSOR_FULL, "CursorFullError"}, - {MDB_PAGE_FULL, "PageFullError"}, - {MDB_MAP_RESIZED, "MapResizedError"}, - {MDB_INCOMPATIBLE, "IncompatibleError"}, - {MDB_BAD_RSLOT, "BadRSlotError"}, - {MDB_BAD_DBI, "BadDbiError"}, - {MDB_BAD_TXN, "BadTxnError"}, - {MDB_BAD_VALSIZE, "BadValsizeError"}, - {EACCES, "ReadonlyError"}, - {EINVAL, "InvalidParameterError"}, - {EAGAIN, "LockError"}, - {ENOMEM, "MemoryError"}, - {ENOSPC, "DiskError"} -}; - - -/* ---------- */ -/* Exceptions */ -/* ---------- */ - -/** - * Raise an exception appropriate for the given `rc` MDB error code. - */ -static void * NOINLINE -err_set(const char *what, int rc) -{ - size_t count = sizeof error_map / sizeof error_map[0]; - PyObject *klass = Error; - size_t i; - - if(rc) { - for(i = 0; i < count; i++) { - if(error_map[i].code == rc) { - klass = error_tbl[i]; - break; - } - } - } - - PyErr_Format(klass, "%s: %s", what, mdb_strerror(rc)); - return NULL; -} - -/** - * Raise an exception from a format string. - */ -static void * NOINLINE -err_format(int rc, const char *fmt, ...) -{ - char buf[128]; - va_list ap; - va_start(ap, fmt); - vsnprintf(buf, sizeof buf, fmt, ap); - buf[sizeof buf - 1] = '\0'; - va_end(ap); - return err_set(buf, rc); -} - -static void * NOINLINE -err_invalid(void) -{ - PyErr_Format(Error, "Attempt to operate on closed/deleted/dropped object."); - return NULL; -} - -static void * NOINLINE -type_error(const char *what) -{ - PyErr_Format(PyExc_TypeError, "%s", what); - return NULL; -} - -/** - * Convert a PyObject to filesystem bytes. Must call fspath_fini() when done. - * Return 0 on success, or set an exception and return -1 on failure. - */ -static PyObject * -get_fspath(PyObject *src) -{ - if(PyBytes_CheckExact(src)) { - Py_INCREF(src); - return src; - } - if(! PyUnicode_CheckExact(src)) { - type_error("Filesystem path must be Unicode or bytes."); - return NULL; - } - return PyUnicode_AsEncodedString(src, Py_FileSystemDefaultEncoding, - "strict"); -} - -/* ------- */ -/* Helpers */ -/* ------- */ - -/** - * Describes the type of a struct field. - */ -enum field_type { - /** Last field in set, stop converting. */ - TYPE_EOF, - /** Unsigned 32bit integer. */ - TYPE_UINT, - /** size_t */ - TYPE_SIZE, - /** void pointer */ - TYPE_ADDR -}; - -/** - * Describes a struct field. - */ -struct dict_field { - /** Field type. */ - enum field_type type; - /** Field name in target dict. */ - const char *name; - /* Offset into structure where field is found. */ - int offset; -}; - -/** - * Return a new reference to Py_True if the given argument is true, otherwise - * a new reference to Py_False. - */ -static PyObject * -py_bool(int pred) -{ - PyObject *obj = pred ? Py_True : Py_False; - Py_INCREF(obj); - return obj; -} - -/** - * Convert the structure `o` described by `fields` to a dict and return the new - * dict. - */ -static PyObject * -dict_from_fields(void *o, const struct dict_field *fields) -{ - PyObject *dict = PyDict_New(); - if(! dict) { - return NULL; - } - - while(fields->type != TYPE_EOF) { - uint8_t *p = ((uint8_t *) o) + fields->offset; - unsigned PY_LONG_LONG l = 0; - PyObject *lo; - - if(fields->type == TYPE_UINT) { - l = *(unsigned int *)p; - } else if(fields->type == TYPE_SIZE) { - l = *(size_t *)p; - } else if(fields->type == TYPE_ADDR) { - l = (intptr_t) *(void **)p; - } - - if(! ((lo = PyLong_FromUnsignedLongLong(l)))) { - Py_DECREF(dict); - return NULL; - } - - if(PyDict_SetItemString(dict, fields->name, lo)) { - Py_DECREF(lo); - Py_DECREF(dict); - return NULL; - } - Py_DECREF(lo); - fields++; - } - return dict; -} - -/** - * Given an MDB_val `val`, convert it to a Python string or bytes object, - * depending on the Python version. Returns a new reference to the object on - * sucess, or NULL on failure. - */ -static PyObject * -obj_from_val(MDB_val *val, int as_buffer) -{ - if(as_buffer) { - return PyMemoryView_FromMemory(val->mv_data, val->mv_size, PyBUF_READ); - } - return PyBytes_FromStringAndSize(val->mv_data, val->mv_size); -} - -/** - * Given some Python object, try to get at its raw data. For string or bytes - * objects, this is the object value. For Unicode objects, this is the UTF-8 - * representation of the object value. For all other objects, attempt to invoke - * the Python 2.x buffer protocol. - */ -static int NOINLINE -val_from_buffer(MDB_val *val, PyObject *buf) -{ - if(PyBytes_CheckExact(buf)) { - val->mv_data = PyBytes_AS_STRING(buf); - val->mv_size = PyBytes_GET_SIZE(buf); - return 0; - } - if(PyUnicode_CheckExact(buf)) { - type_error("Won't implicitly convert Unicode to bytes; use .encode()"); - return -1; - } - return PyObject_AsReadBuffer(buf, - (const void **) &val->mv_data, - (Py_ssize_t *) &val->mv_size); -} - -/* ------------------- */ -/* Concurrency control */ -/* ------------------- */ - -#define UNLOCKED(out, e) \ - Py_BEGIN_ALLOW_THREADS \ - out = (e); \ - Py_END_ALLOW_THREADS - - -/* ---------------- */ -/* Argument parsing */ -/* ---------------- */ - -#define OFFSET(k, y) offsetof(struct k, y) -#define SPECSIZE() (sizeof(argspec) / sizeof(argspec[0])) -enum arg_type { - ARG_DB, /** DbObject* */ - ARG_TRANS, /** TransObject* */ - ARG_ENV, /** EnvObject* */ - ARG_OBJ, /** PyObject* */ - ARG_BOOL, /** int */ - ARG_BUF, /** MDB_val */ - ARG_STR, /** char* */ - ARG_INT, /** int */ - ARG_SIZE /** size_t */ -}; -struct argspec { - const char *string; - unsigned short type; - unsigned short offset; -}; - -static PyTypeObject *type_tbl[] = { - &PyDatabase_Type, - &PyTransaction_Type, - &PyEnvironment_Type -}; - - -static int NOINLINE -parse_ulong(PyObject *obj, uint64_t *l, PyObject *max) -{ - int rc = PyObject_RichCompareBool(obj, py_zero, Py_GE); - if(rc == -1) { - return -1; - } else if(! rc) { - PyErr_Format(PyExc_OverflowError, "Integer argument must be >= 0"); - return -1; - } - rc = PyObject_RichCompareBool(obj, max, Py_LE); - if(rc == -1) { - return -1; - } else if(! rc) { - PyErr_Format(PyExc_OverflowError, "Integer argument exceeds limit."); - return -1; - } -#if PY_MAJOR_VERSION >= 3 - *l = PyLong_AsUnsignedLongLongMask(obj); -#else - *l = PyInt_AsUnsignedLongLongMask(obj); -#endif - return 0; -} - -/** - * Parse a single argument specified by `spec` into `out`, returning 0 on - * success or setting an exception and returning -1 on error. - */ -static int -parse_arg(const struct argspec *spec, PyObject *val, void *out) -{ - void *dst = ((uint8_t *)out) + spec->offset; - int ret = 0; - uint64_t l; - - if(val != Py_None) { - switch((enum arg_type) spec->type) { - case ARG_DB: - case ARG_TRANS: - case ARG_ENV: - if(val->ob_type != type_tbl[spec->type]) { - type_error("invalid type"); - return -1; - } - /* fallthrough */ - case ARG_OBJ: - *((PyObject **) dst) = val; - break; - case ARG_BOOL: - *((int *)dst) = val == Py_True; - break; - case ARG_BUF: - ret = val_from_buffer((MDB_val *)dst, val); - break; - case ARG_STR: { - MDB_val mv; - if(! (ret = val_from_buffer(&mv, val))) { - *((char **) dst) = mv.mv_data; - } - break; - } - case ARG_INT: - if(! (ret = parse_ulong(val, &l, py_int_max))) { - *((int *) dst) = (int)l; - } - break; - case ARG_SIZE: - if(! (ret = parse_ulong(val, &l, py_size_max))) { - *((size_t *) dst) = (size_t)l; - } - break; - } - } - return ret; -} - -/** - * Walk `argspec`, building a Python dictionary mapping keyword arguments to a - * PyInt describing their offset in the array. Used to reduce keyword argument - * parsing from O(specsize) to O(number of supplied kwargs). - */ -static int NOINLINE -make_arg_cache(int specsize, const struct argspec *argspec, PyObject **cache) -{ - Py_ssize_t i; - - if(! ((*cache = PyDict_New()))) { - return -1; - } - - for(i = 0; i < specsize; i++) { - const struct argspec *spec = argspec + i; - PyObject *key = PyUnicode_InternFromString(spec->string); - PyObject *val = MAKE_ID(i); - if((! (key && val)) || PyDict_SetItem(*cache, key, val)) { - return -1; - } - Py_DECREF(val); - } - return 0; -} - -/** - * Like PyArg_ParseTupleAndKeywords except types are specialized for this - * module, keyword strings aren't dup'd every call and the code is >3x smaller. - */ -static int NOINLINE -parse_args(int valid, int specsize, const struct argspec *argspec, - PyObject **cache, PyObject *args, PyObject *kwds, void *out) -{ - unsigned set = 0; - unsigned i; - - if(! valid) { - err_invalid(); - return -1; - } - - if(args) { - int size = (int) PyTuple_GET_SIZE(args); - if(size > specsize) { - type_error("too many positional arguments."); - return -1; - } - if(specsize < size) { - size = specsize; - } - for(i = 0; i < size; i++) { - if(parse_arg(argspec + i, PyTuple_GET_ITEM(args, i), out)) { - return -1; - } - set |= 1 << i; - } - } - - if(kwds) { - Py_ssize_t ppos = 0; - PyObject *pkey; - PyObject *pvalue; - - if((! *cache) && make_arg_cache(specsize, argspec, cache)) { - return -1; - } - - while(PyDict_Next(kwds, &ppos, &pkey, &pvalue)) { - PyObject *specidx; - int i; - - if(! ((specidx = PyDict_GetItem(*cache, pkey)))) { - type_error("unrecognized keyword argument"); - return -1; - } - - i = READ_ID(specidx); - if(set & (1 << i)) { - PyErr_Format(PyExc_TypeError, "duplicate argument: %s", - PyBytes_AS_STRING(pkey)); - return -1; - } - - if(parse_arg(argspec + i, pvalue, out)) { - return -1; - } - } - } - return 0; -} - -/** - * Return 1 if `db` is associated with the given `env`, otherwise raise an - * exception. Used to prevent DBIs from unrelated envs from being mixed - * together (which in future, would cause one env to access another's cursor - * pointers). - */ -static int -db_owner_check(DbObject *db, EnvObject *env) -{ - if(db->env != env) { - err_set("Database handle belongs to another environment.", 0); - return 0; - } - return 1; -} - - -/* -------------------------------------------------------- */ -/* Functionality shared between Transaction and Environment */ -/* -------------------------------------------------------- */ - -static PyObject * -make_trans(EnvObject *env, DbObject *db, TransObject *parent, int write, int buffers) -{ - MDB_txn *parent_txn; - TransObject *self; - int flags; - int rc; - - DEBUG("make_trans(env=%p, parent=%p, write=%d, buffers=%d)", - env, parent, write, buffers) - if(! env->valid) { - return err_invalid(); - } - - if(! db) { - db = env->main_db; - } else if(! db_owner_check(db, env)) { - return NULL; - } - - parent_txn = NULL; - if(parent) { - if(parent->flags & TRANS_RDONLY) { - return err_set("Read-only transactions cannot be nested.", EINVAL); - } - if(! parent->valid) { - return err_invalid(); - } - parent_txn = parent->txn; - } - - if((!write) && env->spare_txns) { - self = env->spare_txns; - DEBUG("found freelist txn; self=%p self->txn=%p", self, self->txn) - env->spare_txns = self->spare_next; - env->max_spare_txns++; - self->flags &= ~TRANS_SPARE; - _Py_NewReference(self); - UNLOCKED(rc, mdb_txn_renew(self->txn)); - - if(rc) { - mdb_txn_abort(self->txn); - self->txn = NULL; - } - } else { - if(write && env->readonly) { - const char *msg = "Cannot start write transaction with read-only env"; - return err_set(msg, EACCES); - } - - if(! ((self = PyObject_New(TransObject, &PyTransaction_Type)))) { - return NULL; - } - - flags = write ? 0 : MDB_RDONLY; - UNLOCKED(rc, mdb_txn_begin(env->env, parent_txn, flags, &self->txn)); - } - - if(rc) { - PyObject_Del(self); - return err_set("mdb_txn_begin", rc); - } - - OBJECT_INIT(self) - LINK_CHILD(env, self) - self->weaklist = NULL; - self->env = env; - Py_INCREF(env); - self->db = db; - Py_INCREF(db); -#ifdef HAVE_MEMSINK - self->sink_head = NULL; -#endif - - self->mutations = 0; - self->spare_next = NULL; - self->flags = 0; - if(! write) { - self->flags |= TRANS_RDONLY; - } - if(buffers) { - self->flags |= TRANS_BUFFERS; - } - return (PyObject *)self; -} - -static PyObject * -make_cursor(DbObject *db, TransObject *trans) -{ - CursorObject *self; - int rc; - - if(! trans->valid) { - return err_invalid(); - } - if(! db) { - db = trans->env->main_db; - } else if(! db_owner_check(db, trans->env)) { - return NULL; - } - - self = PyObject_New(CursorObject, &PyCursor_Type); - UNLOCKED(rc, mdb_cursor_open(trans->txn, db->dbi, &self->curs)); - if(rc) { - PyObject_Del(self); - return err_set("mdb_cursor_open", rc); - } - - DEBUG("sizeof cursor = %d", (int) sizeof *self) - OBJECT_INIT(self) - LINK_CHILD(trans, self) - self->positioned = 0; - self->key.mv_size = 0; - self->val.mv_size = 0; - self->trans = trans; - self->last_mutation = trans->mutations; - self->dbi_flags = db->flags; - Py_INCREF(self->trans); - return (PyObject *) self; -} - - -/* -------- */ -/* Database */ -/* -------- */ - -static DbObject * -db_from_name(EnvObject *env, MDB_txn *txn, const char *name, - unsigned int flags) -{ - MDB_dbi dbi; - unsigned int f; - int rc; - DbObject *dbo; - - UNLOCKED(rc, mdb_dbi_open(txn, name, flags, &dbi)); - if(rc) { - err_set("mdb_dbi_open", rc); - return NULL; - } - if((rc = mdb_dbi_flags(txn, dbi, &f))) { - err_set("mdb_dbi_flags", rc); - mdb_dbi_close(env->env, dbi); - return NULL; - } - - if(! ((dbo = PyObject_New(DbObject, &PyDatabase_Type)))) { - return NULL; - } - - OBJECT_INIT(dbo) - LINK_CHILD(env, dbo) - dbo->env = env; /* no refcount */ - dbo->dbi = dbi; - dbo->flags = f; - DEBUG("DbObject '%s' opened at %p", name, dbo) - return dbo; -} - -/** - * Use a temporary transaction to manufacture a new _Database object for - * `name`. - */ -static DbObject * -txn_db_from_name(EnvObject *env, const char *name, - unsigned int flags) -{ - int rc; - MDB_txn *txn; - DbObject *dbo; - - int begin_flags = (name == NULL || env->readonly) ? MDB_RDONLY : 0; - UNLOCKED(rc, mdb_txn_begin(env->env, NULL, begin_flags, &txn)); - if(rc) { - err_set("mdb_txn_begin", rc); - return NULL; - } - - if(! ((dbo = db_from_name(env, txn, name, flags)))) { - Py_BEGIN_ALLOW_THREADS - mdb_txn_abort(txn); - Py_END_ALLOW_THREADS - return NULL; - } - - UNLOCKED(rc, mdb_txn_commit(txn)); - if(rc) { - Py_DECREF(dbo); - return err_set("mdb_txn_commit", rc); - } - return dbo; -} - -static int -db_clear(DbObject *self) -{ - if(self->env) { - UNLINK_CHILD(self->env, self) - self->env = NULL; - } - self->valid = 0; - return 0; -} - -/** - * _Database.flags() - */ -static PyObject * -db_flags(DbObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *dct; - unsigned int f; - - struct db_flags { - TransObject *txn; - } arg = {NULL}; - - static const struct argspec argspec[] = { - {"txn", ARG_TRANS, OFFSET(db_flags, txn)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! arg.txn) { - return type_error("'txn' argument required"); - } - if(! arg.txn->valid) { - return err_invalid(); - } - - dct = PyDict_New(); - f = self->flags; - PyDict_SetItemString(dct, "reverse_key", py_bool(f & MDB_REVERSEKEY)); - PyDict_SetItemString(dct, "dupsort", py_bool(f & MDB_DUPSORT)); - return dct; -} - -/** - * _Database.__del__() - */ -static void -db_dealloc(DbObject *self) -{ - db_clear(self); - PyObject_Del(self); -} - -static struct PyMethodDef db_methods[] = { - {"flags", (PyCFunction)db_flags, METH_VARARGS|METH_KEYWORDS}, - {0, 0, 0, 0} -}; - -static PyTypeObject PyDatabase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_Database", /*tp_name*/ - sizeof(DbObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)db_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - (inquiry)db_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - db_methods /*tp_methods*/ -}; - - -/* ----------- */ -/* Environment */ -/* ----------- */ - -static void -trans_dealloc(TransObject *self); - -static int -env_clear(EnvObject *self) -{ - MDEBUG("killing env..") - - INVALIDATE(self) - self->valid = 0; - Py_CLEAR(self->main_db); - - /* Force trans_dealloc() to free by setting avail size to 0 */ - self->max_spare_txns = 0; - while(self->spare_txns) { - TransObject *cur = self->spare_txns; - MDEBUG("killing spare txn %p", self->spare_txns) - self->spare_txns = cur->spare_next; - trans_dealloc(cur); - } - - if(self->env) { - DEBUG("Closing env") - Py_BEGIN_ALLOW_THREADS - mdb_env_close(self->env); - Py_END_ALLOW_THREADS - self->env = NULL; - } - return 0; -} - -/** - * Environment.__del__() - */ -static void -env_dealloc(EnvObject *self) -{ - if(self->weaklist != NULL) { - MDEBUG("Clearing weaklist..") - PyObject_ClearWeakRefs((PyObject *) self); - } - - env_clear(self); - PyObject_Del(self); -} - -/** - * Environment.close() - */ -static PyObject * -env_close(EnvObject *self) -{ - env_clear(self); - Py_RETURN_NONE; -} - -/** - * Environment() -> new object. - */ -static PyObject * -env_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - struct env_new { - PyObject *path; - size_t map_size; - int subdir; - int readonly; - int metasync; - int sync; - int map_async; - int mode; - int create; - int readahead; - int writemap; - int meminit; - int max_readers; - int max_dbs; - int max_spare_txns; - int lock; - } arg = {NULL, 10485760, 1, 0, 1, 1, 0, 0755, 1, 1, 0, 1, 126, 0, 1, 1}; - - static const struct argspec argspec[] = { - {"path", ARG_OBJ, OFFSET(env_new, path)}, - {"map_size", ARG_SIZE, OFFSET(env_new, map_size)}, - {"subdir", ARG_BOOL, OFFSET(env_new, subdir)}, - {"readonly", ARG_BOOL, OFFSET(env_new, readonly)}, - {"metasync", ARG_BOOL, OFFSET(env_new, metasync)}, - {"sync", ARG_BOOL, OFFSET(env_new, sync)}, - {"map_async", ARG_BOOL, OFFSET(env_new, map_async)}, - {"mode", ARG_INT, OFFSET(env_new, mode)}, - {"create", ARG_BOOL, OFFSET(env_new, create)}, - {"readahead", ARG_BOOL, OFFSET(env_new, readahead)}, - {"writemap", ARG_BOOL, OFFSET(env_new, writemap)}, - {"meminit", ARG_INT, OFFSET(env_new, meminit)}, - {"max_readers", ARG_INT, OFFSET(env_new, max_readers)}, - {"max_dbs", ARG_INT, OFFSET(env_new, max_dbs)}, - {"max_spare_txns", ARG_INT, OFFSET(env_new, max_spare_txns)}, - {"lock", ARG_BOOL, OFFSET(env_new, lock)} - }; - - PyObject *fspath_obj = NULL; - EnvObject *self; - const char *fspath; - int flags; - int rc; - int mode; - - static PyObject *cache = NULL; - if(parse_args(1, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - if(! arg.path) { - return type_error("'path' argument required"); - } - - if(! ((self = PyObject_New(EnvObject, type)))) { - return NULL; - } - - OBJECT_INIT(self) - self->weaklist = NULL; - self->main_db = NULL; - self->env = NULL; - self->max_spare_txns = arg.max_spare_txns; - self->spare_txns = NULL; - - if((rc = mdb_env_create(&self->env))) { - err_set("mdb_env_create", rc); - goto fail; - } - - if((rc = mdb_env_set_mapsize(self->env, arg.map_size))) { - err_set("mdb_env_set_mapsize", rc); - goto fail; - } - - if((rc = mdb_env_set_maxreaders(self->env, arg.max_readers))) { - err_set("mdb_env_set_maxreaders", rc); - goto fail; - } - - if((rc = mdb_env_set_maxdbs(self->env, arg.max_dbs))) { - err_set("mdb_env_set_maxdbs", rc); - goto fail; - } - - if(! ((fspath_obj = get_fspath(arg.path)))) { - goto fail; - } - fspath = PyBytes_AS_STRING(fspath_obj); - - if(arg.create && arg.subdir && !arg.readonly) { - struct stat st; - errno = 0; - stat(fspath, &st); - if(errno == ENOENT) { - if(mkdir(fspath, arg.mode)) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, fspath); - goto fail; - } - } - } - - flags = MDB_NOTLS; - if(! arg.subdir) { - flags |= MDB_NOSUBDIR; - } - if(arg.readonly) { - flags |= MDB_RDONLY; - } - self->readonly = arg.readonly; - if(! arg.metasync) { - flags |= MDB_NOMETASYNC; - } - if(! arg.sync) { - flags |= MDB_NOSYNC; - } - if(arg.map_async) { - flags |= MDB_MAPASYNC; - } - if(! arg.readahead) { - flags |= MDB_NORDAHEAD; - } - if(arg.writemap) { - flags |= MDB_WRITEMAP; - } - if(! arg.meminit) { - flags |= MDB_NOMEMINIT; - } - if(! arg.lock) { - flags |= MDB_NOLOCK; - } - - /* Strip +x. */ - mode = arg.mode & ~0111; - - DEBUG("mdb_env_open(%p, '%s', %d, %o);", self->env, fspath, flags, mode) - UNLOCKED(rc, mdb_env_open(self->env, fspath, flags, mode)); - if(rc) { - err_set(fspath, rc); - goto fail; - } - - self->main_db = txn_db_from_name(self, NULL, 0); - if(self->main_db) { - self->valid = 1; - DEBUG("EnvObject '%s' opened at %p", fspath, self) - return (PyObject *) self; - } - -fail: - DEBUG("initialization failed") - Py_CLEAR(fspath_obj); - Py_CLEAR(self); - return NULL; -} - -/** - * Environment.begin() - */ -static PyObject * -env_begin(EnvObject *self, PyObject *args, PyObject *kwds) -{ - struct env_begin { - DbObject *db; - TransObject *parent; - int write; - int buffers; - } arg = {self->main_db, NULL, 0, 0}; - - static const struct argspec argspec[] = { - {"db", ARG_DB, OFFSET(env_begin, db)}, - {"parent", ARG_TRANS, OFFSET(env_begin, parent)}, - {"write", ARG_BOOL, OFFSET(env_begin, write)}, - {"buffers", ARG_BOOL, OFFSET(env_begin, buffers)}, - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - return make_trans(self, arg.db, arg.parent, arg.write, arg.buffers); -} - -/** - * Environment.copy() - */ -static PyObject * -env_copy(EnvObject *self, PyObject *args, PyObject *kwds) -{ - struct env_copy { - PyObject *path; - int compact; - } arg = {NULL, 0}; - - static const struct argspec argspec[] = { - {"path", ARG_OBJ, OFFSET(env_copy, path)}, - {"compact", ARG_BOOL, OFFSET(env_copy, compact)} - }; - - PyObject *fspath_obj; - const char *fspath_s; - int flags; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! arg.path) { - return type_error("path argument required"); - } - if(! ((fspath_obj = get_fspath(arg.path)))) { - return NULL; - } - - fspath_s = PyBytes_AS_STRING(fspath_obj); - flags = arg.compact ? MDB_CP_COMPACT : 0; - UNLOCKED(rc, mdb_env_copy2(self->env, fspath_s, flags)); - Py_CLEAR(fspath_obj); - if(rc) { - return err_set("mdb_env_copy2", rc); - } - Py_RETURN_NONE; -} - -/** - * Environment.copyfd(fd) - */ -static PyObject * -env_copyfd(EnvObject *self, PyObject *args, PyObject *kwds) -{ - struct env_copyfd { - int fd; - int compact; - } arg = {-1, 0}; - int rc; -#ifdef _WIN32 - PyObject *temp; - Py_ssize_t handle; -#endif - - static const struct argspec argspec[] = { - {"fd", ARG_INT, OFFSET(env_copyfd, fd)}, - {"compact", ARG_BOOL, OFFSET(env_copyfd, compact)} - }; - int flags; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(arg.fd == -1) { - return type_error("fd argument required"); - } - flags = arg.compact ? MDB_CP_COMPACT : 0; - -#ifdef _WIN32 - temp = PyObject_CallMethod(msvcrt, "get_osfhandle", "i", arg.fd); - if(! temp) { - return NULL; - } - handle = PyNumber_AsSsize_t(temp, PyExc_OverflowError); - Py_DECREF(temp); - if(PyErr_Occurred()) { - return NULL; - } - UNLOCKED(rc, mdb_env_copyfd2(self->env, (HANDLE)handle, flags)); -#else - UNLOCKED(rc, mdb_env_copyfd2(self->env, arg.fd, flags)); -#endif - - if(rc) { - return err_set("mdb_env_copyfd2", rc); - } - Py_RETURN_NONE; -} - -/** - * Environment.info() -> dict - */ -static PyObject * -env_info(EnvObject *self) -{ - static const struct dict_field fields[] = { - {TYPE_ADDR, "map_addr", offsetof(MDB_envinfo, me_mapaddr)}, - {TYPE_SIZE, "map_size", offsetof(MDB_envinfo, me_mapsize)}, - {TYPE_SIZE, "last_pgno", offsetof(MDB_envinfo, me_last_pgno)}, - {TYPE_SIZE, "last_txnid", offsetof(MDB_envinfo, me_last_txnid)}, - {TYPE_UINT, "max_readers", offsetof(MDB_envinfo, me_maxreaders)}, - {TYPE_UINT, "num_readers", offsetof(MDB_envinfo, me_numreaders)}, - {TYPE_EOF, NULL, 0} - }; - MDB_envinfo info; - int rc; - - if(! self->valid) { - return err_invalid(); - } - - UNLOCKED(rc, mdb_env_info(self->env, &info)); - if(rc) { - err_set("mdb_env_info", rc); - return NULL; - } - return dict_from_fields(&info, fields); -} - -/** - * Environment.flags() -> dict - */ -static PyObject * -env_flags(EnvObject *self) -{ - PyObject *dct; - unsigned int flags; - int rc; - - if(! self->valid) { - return err_invalid(); - } - - if((rc = mdb_env_get_flags(self->env, &flags))) { - err_set("mdb_env_get_flags", rc); - return NULL; - } - - dct = PyDict_New(); - PyDict_SetItemString(dct, "subdir", py_bool(!(flags & MDB_NOSUBDIR))); - PyDict_SetItemString(dct, "readonly", py_bool(flags & MDB_RDONLY)); - PyDict_SetItemString(dct, "metasync", py_bool(!(flags & MDB_NOMETASYNC))); - PyDict_SetItemString(dct, "sync", py_bool(!(flags & MDB_NOSYNC))); - PyDict_SetItemString(dct, "map_async", py_bool(flags & MDB_MAPASYNC)); - PyDict_SetItemString(dct, "readahead", py_bool(!(flags & MDB_NORDAHEAD))); - PyDict_SetItemString(dct, "writemap", py_bool(flags & MDB_WRITEMAP)); - PyDict_SetItemString(dct, "meminit", py_bool(!(flags & MDB_NOMEMINIT))); - PyDict_SetItemString(dct, "lock", py_bool(!(flags & MDB_NOLOCK))); - return dct; -} - -/** - * Environment.max_key_size() -> int - */ -static PyObject * -env_max_key_size(EnvObject *self) -{ - int key_size; - if(! self->valid) { - return err_invalid(); - } - key_size = mdb_env_get_maxkeysize(self->env); - return PyLong_FromLongLong(key_size); -} - -/** - * Environment.max_key_size() -> int - */ -static PyObject * -env_max_readers(EnvObject *self) -{ - unsigned int readers; - int rc; - - if(! self->valid) { - return err_invalid(); - } - if((rc = mdb_env_get_maxreaders(self->env, &readers))) { - return err_set("mdb_env_get_maxreaders", rc); - } - return PyLong_FromLongLong(readers); -} - -/** - * Environment.open_db() -> handle - */ -static PyObject * -env_open_db(EnvObject *self, PyObject *args, PyObject *kwds) -{ - struct env_open_db { - const char *key; - TransObject *txn; - int reverse_key; - int dupsort; - int create; - } arg = {NULL, NULL, 0, 0, 1}; - - static const struct argspec argspec[] = { - {"key", ARG_STR, OFFSET(env_open_db, key)}, - {"txn", ARG_TRANS, OFFSET(env_open_db, txn)}, - {"reverse_key", ARG_BOOL, OFFSET(env_open_db, reverse_key)}, - {"dupsort", ARG_BOOL, OFFSET(env_open_db, dupsort)}, - {"create", ARG_BOOL, OFFSET(env_open_db, create)}, - }; - int flags; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - flags = 0; - if(arg.reverse_key) { - flags |= MDB_REVERSEKEY; - } - if(arg.dupsort) { - flags |= MDB_DUPSORT; - } - if(arg.create) { - flags |= MDB_CREATE; - } - - if(arg.txn) { - return (PyObject *) db_from_name(self, arg.txn->txn, arg.key, flags); - } else { - return (PyObject *) txn_db_from_name(self, arg.key, flags); - } -} - -/** - * Environment.path() -> Unicode - */ -static PyObject * -env_path(EnvObject *self) -{ - const char *path; - int rc; - - if(! self->valid) { - return err_invalid(); - } - - if((rc = mdb_env_get_path(self->env, &path))) { - return err_set("mdb_env_get_path", rc); - } - return PyUnicode_FromString(path); -} - -static const struct dict_field mdb_stat_fields[] = { - {TYPE_UINT, "psize", offsetof(MDB_stat, ms_psize)}, - {TYPE_UINT, "depth", offsetof(MDB_stat, ms_depth)}, - {TYPE_SIZE, "branch_pages", offsetof(MDB_stat, ms_branch_pages)}, - {TYPE_SIZE, "leaf_pages", offsetof(MDB_stat, ms_leaf_pages)}, - {TYPE_SIZE, "overflow_pages", offsetof(MDB_stat, ms_overflow_pages)}, - {TYPE_SIZE, "entries", offsetof(MDB_stat, ms_entries)}, - {TYPE_EOF, NULL, 0} -}; - -/** - * Environment.stat() -> dict - */ -static PyObject * -env_stat(EnvObject *self) -{ - MDB_stat st; - int rc; - - if(! self->valid) { - return err_invalid(); - } - - UNLOCKED(rc, mdb_env_stat(self->env, &st)); - if(rc) { - err_set("mdb_env_stat", rc); - return NULL; - } - return dict_from_fields(&st, mdb_stat_fields); -} - -/** - * Callback to receive string result for env_readers(). Return 0 on success or - * -1 on error. - */ -static int env_readers_callback(const char *msg, void *str_) -{ - PyObject **str = str_; - PyObject *s = PyUnicode_FromString(msg); - PyObject *new; - if(! s) { - return -1; - } - new = PyUnicode_Concat(*str, s); - Py_CLEAR(*str); - *str = new; - if(! new) { - return -1; - } - return 0; -} - -/** - * Environment.readers() -> string - */ -static PyObject * -env_readers(EnvObject *self) -{ - PyObject *str; - if(! self->valid) { - return err_invalid(); - } - - if(! ((str = PyUnicode_FromString("")))) { - return NULL; - } - - if(mdb_reader_list(self->env, env_readers_callback, &str)) { - Py_CLEAR(str); - } - return str; -} - -/** - * Environment.reader_check() -> int - */ -static PyObject * -env_reader_check(EnvObject *self) -{ - int rc; - int dead; - - if(! self->valid) { - return err_invalid(); - } - - if((rc = mdb_reader_check(self->env, &dead))) { - return err_set("mdb_reader_check", rc); - } - return PyLong_FromLongLong(dead); -} - -/** - * Environment.sync() - */ -static PyObject * -env_sync(EnvObject *self, PyObject *args) -{ - struct env_sync { - int force; - } arg = {0}; - - static const struct argspec argspec[] = { - {"force", ARG_BOOL, OFFSET(env_sync, force)} - }; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, NULL, &arg)) { - return NULL; - } - - UNLOCKED(rc, mdb_env_sync(self->env, arg.force)); - if(rc) { - return err_set("mdb_env_sync", rc); - } - Py_RETURN_NONE; -} - -/** - * Environment.__enter__() - */ -static PyObject *env_enter(EnvObject *self) -{ - Py_INCREF(self); - return (PyObject *)self; -} - -/** - * Environment.__exit__() - */ -static PyObject *env_exit(EnvObject *self, PyObject *args) -{ - env_clear(self); - Py_RETURN_NONE; -} - -static struct PyMethodDef env_methods[] = { - {"__enter__", (PyCFunction)env_enter, METH_NOARGS}, - {"__exit__", (PyCFunction)env_exit, METH_VARARGS}, - {"begin", (PyCFunction)env_begin, METH_VARARGS|METH_KEYWORDS}, - {"close", (PyCFunction)env_close, METH_NOARGS}, - {"copy", (PyCFunction)env_copy, METH_VARARGS|METH_KEYWORDS}, - {"copyfd", (PyCFunction)env_copyfd, METH_VARARGS|METH_KEYWORDS}, - {"info", (PyCFunction)env_info, METH_NOARGS}, - {"flags", (PyCFunction)env_flags, METH_NOARGS}, - {"max_key_size", (PyCFunction)env_max_key_size, METH_NOARGS}, - {"max_readers", (PyCFunction)env_max_readers, METH_NOARGS}, - {"open_db", (PyCFunction)env_open_db, METH_VARARGS|METH_KEYWORDS}, - {"path", (PyCFunction)env_path, METH_NOARGS}, - {"stat", (PyCFunction)env_stat, METH_NOARGS}, - {"readers", (PyCFunction)env_readers, METH_NOARGS}, - {"reader_check", (PyCFunction)env_reader_check, METH_NOARGS}, - {"sync", (PyCFunction)env_sync, METH_VARARGS}, - {NULL, NULL} -}; - -static PyTypeObject PyEnvironment_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "Environment", /*tp_name*/ - sizeof(EnvObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor) env_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - (inquiry) env_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(EnvObject, weaklist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - env_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - env_new, /*tp_new*/ -}; - - -/* ------- */ -/* Cursors */ -/* ------- */ - -static int -cursor_clear(CursorObject *self) -{ - if(self->valid) { - INVALIDATE(self) - UNLINK_CHILD(self->trans, self) - Py_BEGIN_ALLOW_THREADS - mdb_cursor_close(self->curs); - Py_END_ALLOW_THREADS - self->valid = 0; - } - Py_CLEAR(self->trans); - return 0; -} - -/** - * Cursor.__del__() - */ -static void -cursor_dealloc(CursorObject *self) -{ - DEBUG("destroying cursor") - cursor_clear(self); - PyObject_Del(self); -} - -/** - * Environment.Cursor(db, trans) -> new instance. - */ -static PyObject * -cursor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - struct cursor_new { - DbObject *db; - TransObject *trans; - } arg = {NULL, NULL}; - - static const struct argspec argspec[] = { - {"db", ARG_DB, OFFSET(cursor_new, db)}, - {"txn", ARG_TRANS, OFFSET(cursor_new, trans)} - }; - - static PyObject *cache = NULL; - if(parse_args(1, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - if(! (arg.db && arg.trans)) { - return type_error("db and transaction parameters required."); - } - return make_cursor(arg.db, arg.trans); -} - -/** - * Cursor.count() -> long - */ -static PyObject * -cursor_count(CursorObject *self) -{ - size_t count; - int rc; - - if(! self->valid) { - return err_invalid(); - } - - UNLOCKED(rc, mdb_cursor_count(self->curs, &count)); - if(rc) { - return err_set("mdb_cursor_count", rc); - } - return PyLong_FromUnsignedLongLong(count); -} - -/** - * Apply `op` to the cursor using the Cursor instance's `key` and `val` - * MDB_vals. On completion, check for an error and if one occurred, set the - * cursor to the unpositioned state. Finally record the containing - * transaction's last mutation count, so we know if `key` and `val` become - * invalid before the next attempt to read them. - */ -static int -_cursor_get_c(CursorObject *self, enum MDB_cursor_op op) -{ - int rc; - - Py_BEGIN_ALLOW_THREADS; - rc = mdb_cursor_get(self->curs, &self->key, &self->val, op); - preload(rc, self->val.mv_data, self->val.mv_size); - Py_END_ALLOW_THREADS; - - self->positioned = rc == 0; - self->last_mutation = self->trans->mutations; - if(rc) { - self->key.mv_size = 0; - self->val.mv_size = 0; - if(rc != MDB_NOTFOUND) { - if(! (rc == EINVAL && op == MDB_GET_CURRENT)) { - err_set("mdb_cursor_get", rc); - return -1; - } - } - } - return 0; -} - -/** - * Wrap _cursor_get_c() to return True or False depending on whether the - * Cursor's final state is positioned. - */ -static PyObject * -_cursor_get(CursorObject *self, enum MDB_cursor_op op) -{ - if(! self->valid) { - return err_invalid(); - } - if(_cursor_get_c(self, op)) { - return NULL; - } - return py_bool(self->positioned); -} - -/** - * Cursor.delete(dupdata=False) -> bool - */ -static PyObject * -cursor_delete(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_delete { - int dupdata; - } arg = {0}; - - static const struct argspec argspec[] = { - {"dupdata", ARG_BOOL, OFFSET(cursor_delete, dupdata)} - }; - int res; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - res = 0; - if(self->positioned) { - int rc; - int flags = arg.dupdata ? MDB_NODUPDATA : 0; - DEBUG("deleting key '%.*s'", - (int) self->key.mv_size, - (char*) self->key.mv_data) - UNLOCKED(rc, mdb_cursor_del(self->curs, flags)); - self->trans->mutations++; - if(rc) { - return err_set("mdb_cursor_del", rc); - } - res = 1; - _cursor_get_c(self, MDB_GET_CURRENT); - } - return py_bool(res); -} - -/** - * Cursor.first() -> bool - */ -static PyObject * -cursor_first(CursorObject *self) -{ - return _cursor_get(self, MDB_FIRST); -} - -/** - * Cursor.first_dup() -> bool - */ -static PyObject * -cursor_first_dup(CursorObject *self) -{ - return _cursor_get(self, MDB_FIRST_DUP); -} - -static PyObject * -cursor_value(CursorObject *self); - -/** - * Cursor.get() -> result - */ -static PyObject * -cursor_get(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_get { - MDB_val key; - PyObject *default_; - } arg = {{0, 0}, Py_None}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_get, key)}, - {"default", ARG_OBJ, OFFSET(cursor_get, default_)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - if(! arg.key.mv_data) { - return type_error("key must be given."); - } - - self->key = arg.key; - if(_cursor_get_c(self, MDB_SET_KEY)) { - return NULL; - } - if(! self->positioned) { - Py_INCREF(arg.default_); - return arg.default_; - } - return cursor_value(self); -} - -/** - * Cursor.item() -> (key, value) - */ -static PyObject * -cursor_item(CursorObject *self) -{ - int as_buffer; - PyObject *key; - PyObject *val; - PyObject *tup; - - if(! self->valid) { - return err_invalid(); - } - /* Must refresh `key` and `val` following mutation. */ - if(self->last_mutation != self->trans->mutations && - _cursor_get_c(self, MDB_GET_CURRENT)) { - return NULL; - } - - as_buffer = self->trans->flags & TRANS_BUFFERS; - key = obj_from_val(&self->key, as_buffer); - val = obj_from_val(&self->val, as_buffer); - tup = PyTuple_New(2); - if(tup && key && val) { - PyTuple_SET_ITEM(tup, 0, key); - PyTuple_SET_ITEM(tup, 1, val); - return tup; - } - Py_CLEAR(key); - Py_CLEAR(val); - Py_CLEAR(tup); - return NULL; -} - -/** - * Cursor.key() -> result - */ -static PyObject * -cursor_key(CursorObject *self) -{ - if(! self->valid) { - return err_invalid(); - } - /* Must refresh `key` and `val` following mutation. */ - if(self->last_mutation != self->trans->mutations && - _cursor_get_c(self, MDB_GET_CURRENT)) { - return NULL; - } - return obj_from_val(&self->key, self->trans->flags & TRANS_BUFFERS); -} - -/** - * Cursor.last() -> bool - */ -static PyObject * -cursor_last(CursorObject *self) -{ - return _cursor_get(self, MDB_LAST); -} - -/** - * Cursor.last_dup() -> bool - */ -static PyObject * -cursor_last_dup(CursorObject *self) -{ - return _cursor_get(self, MDB_LAST_DUP); -} - -/** - * Cursor.next() -> bool - */ -static PyObject * -cursor_next(CursorObject *self) -{ - return _cursor_get(self, MDB_NEXT); -} - -/** - * Cursor.next_dup() -> bool - */ -static PyObject * -cursor_next_dup(CursorObject *self) -{ - return _cursor_get(self, MDB_NEXT_DUP); -} - -/** - * Cursor.next_nodup() -> bool - */ -static PyObject * -cursor_next_nodup(CursorObject *self) -{ - return _cursor_get(self, MDB_NEXT_NODUP); -} - -/** - * Cursor.prev() -> bool - */ -static PyObject * -cursor_prev(CursorObject *self) -{ - return _cursor_get(self, MDB_PREV); -} - -/** - * Cursor.prev_dup() -> bool - */ -static PyObject * -cursor_prev_dup(CursorObject *self) -{ - return _cursor_get(self, MDB_PREV_DUP); -} - -/** - * Cursor.prev_nodup() -> bool - */ -static PyObject * -cursor_prev_nodup(CursorObject *self) -{ - return _cursor_get(self, MDB_PREV_NODUP); -} - -/** - * Cursor.putmulti(iter|dict) -> (consumed, added) - */ -static PyObject * -cursor_put_multi(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_put { - PyObject *items; - int dupdata; - int overwrite; - int append; - } arg = {Py_None, 1, 1, 0}; - - PyObject *iter; - PyObject *item; - - static const struct argspec argspec[] = { - {"items", ARG_OBJ, OFFSET(cursor_put, items)}, - {"dupdata", ARG_BOOL, OFFSET(cursor_put, dupdata)}, - {"overwrite", ARG_BOOL, OFFSET(cursor_put, overwrite)}, - {"append", ARG_BOOL, OFFSET(cursor_put, append)} - }; - int flags; - int rc; - Py_ssize_t consumed; - Py_ssize_t added; - PyObject *ret = NULL; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - flags = 0; - if(! arg.dupdata) { - flags |= MDB_NODUPDATA; - } - if(! arg.overwrite) { - flags |= MDB_NOOVERWRITE; - } - if(arg.append) { - flags |= MDB_APPEND; - } - - if(! ((iter = PyObject_GetIter(arg.items)))) { - return NULL; - } - - consumed = 0; - added = 0; - while((item = PyIter_Next(iter))) { - MDB_val mkey, mval; - if(! (PyTuple_CheckExact(item) && PyTuple_GET_SIZE(item) == 2)) { - PyErr_SetString(PyExc_TypeError, - "putmulti() elements must be 2-tuples"); - Py_DECREF(item); - Py_DECREF(iter); - return NULL; - } - - if(val_from_buffer(&mkey, PyTuple_GET_ITEM(item, 0)) || - val_from_buffer(&mval, PyTuple_GET_ITEM(item, 1))) { - Py_DECREF(item); - Py_DECREF(iter); - return NULL; /* val_from_buffer sets exception */ - } - - UNLOCKED(rc, mdb_cursor_put(self->curs, &mkey, &mval, flags)); - self->trans->mutations++; - switch(rc) { - case MDB_SUCCESS: - added++; - break; - case MDB_KEYEXIST: - break; - default: - Py_DECREF(item); - Py_DECREF(iter); - return err_format(rc, "mdb_cursor_put() element #%d", consumed); - } - - Py_DECREF(item); - consumed++; - } - - Py_DECREF(iter); - if(! PyErr_Occurred()) { - ret = Py_BuildValue("(nn)", consumed, added); - } - return ret; -} - -/** - * Cursor.put() -> bool - */ -static PyObject * -cursor_put(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_put { - MDB_val key; - MDB_val val; - int dupdata; - int overwrite; - int append; - } arg = {{0, 0}, {0, 0}, 1, 1, 0}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_put, key)}, - {"value", ARG_BUF, OFFSET(cursor_put, val)}, - {"dupdata", ARG_BOOL, OFFSET(cursor_put, dupdata)}, - {"overwrite", ARG_BOOL, OFFSET(cursor_put, overwrite)}, - {"append", ARG_BOOL, OFFSET(cursor_put, append)} - }; - int flags; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - flags = 0; - if(! arg.dupdata) { - flags |= MDB_NODUPDATA; - } - if(! arg.overwrite) { - flags |= MDB_NOOVERWRITE; - } - if(arg.append) { - flags |= MDB_APPEND; - } - - UNLOCKED(rc, mdb_cursor_put(self->curs, &arg.key, &arg.val, flags)); - self->trans->mutations++; - if(rc) { - if(rc == MDB_KEYEXIST) { - Py_RETURN_FALSE; - } - return err_set("mdb_put", rc); - } - Py_RETURN_TRUE; -} - -/** - * Shared between Cursor.replace() and Transaction.replace() - */ -static PyObject * -do_cursor_replace(CursorObject *self, MDB_val *key, MDB_val *val) -{ - int rc; - PyObject *old; - MDB_val newval = *val; - - if(self->dbi_flags & MDB_DUPSORT) { - self->key = *key; - if(_cursor_get_c(self, MDB_SET_KEY)) { - return NULL; - } - if(self->positioned) { - if(! ((old = obj_from_val(&self->val, 0)))) { - return NULL; - } - UNLOCKED(rc, mdb_cursor_del(self->curs, MDB_NODUPDATA)); - self->trans->mutations++; - if(rc) { - Py_CLEAR(old); - return err_set("mdb_cursor_del", rc); - } - } else { - old = Py_None; - Py_INCREF(old); - } - } else { - /* val is updated if MDB_KEYEXIST. */ - int flags = MDB_NOOVERWRITE; - UNLOCKED(rc, mdb_cursor_put(self->curs, key, val, flags)); - self->trans->mutations++; - if(! rc) { - Py_RETURN_NONE; - } else if(rc != MDB_KEYEXIST) { - return err_set("mdb_put", rc); - } - - if(! ((old = obj_from_val(val, 0)))) { - return NULL; - } - } - - UNLOCKED(rc, mdb_cursor_put(self->curs, key, &newval, 0)); - if(rc) { - Py_DECREF(old); - return err_set("mdb_put", rc); - } - return old; -} - -/** - * Cursor.replace() -> None|result - */ -static PyObject * -cursor_replace(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_replace { - MDB_val key; - MDB_val val; - } arg = {{0, 0}, {0, 0}}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_replace, key)}, - {"value", ARG_BUF, OFFSET(cursor_replace, val)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - return do_cursor_replace(self, &arg.key, &arg.val); -} - -/** - * Cursor.pop() -> None|result - */ -static PyObject * -cursor_pop(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_pop { - MDB_val key; - } arg = {{0, 0}}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_pop, key)}, - }; - PyObject *old; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - self->key = arg.key; - if(_cursor_get_c(self, MDB_SET_KEY)) { - return NULL; - } - if(! self->positioned) { - Py_RETURN_NONE; - } - if(! ((old = obj_from_val(&self->val, 0)))) { - return NULL; - } - - UNLOCKED(rc, mdb_cursor_del(self->curs, 0)); - self->trans->mutations++; - if(rc) { - Py_DECREF(old); - return err_set("mdb_cursor_del", rc); - } - return old; -} - -/** - * Cursor.set_key(key) -> bool - */ -static PyObject * -cursor_set_key(CursorObject *self, PyObject *arg) -{ - if(! self->valid) { - return err_invalid(); - } - if(val_from_buffer(&self->key, arg)) { - return NULL; - } - return _cursor_get(self, MDB_SET_KEY); -} - -/** - * Cursor.set_key_dup(key, value) -> bool - */ -static PyObject * -cursor_set_key_dup(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_set_key_dup { - MDB_val key; - MDB_val value; - } arg = {{0, 0}, {0, 0}}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_set_key_dup, key)}, - {"value", ARG_BUF, OFFSET(cursor_set_key_dup, value)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - self->key = arg.key; - self->val = arg.value; - return _cursor_get(self, MDB_GET_BOTH); -} - -/** - * Cursor.set_range(key) -> bool - */ -static PyObject * -cursor_set_range(CursorObject *self, PyObject *arg) -{ - if(! self->valid) { - return err_invalid(); - } - if(val_from_buffer(&self->key, arg)) { - return NULL; - } - if(self->key.mv_size) { - return _cursor_get(self, MDB_SET_RANGE); - } - return _cursor_get(self, MDB_FIRST); -} - -/** - * Cursor.set_range_dup(key, value) -> bool - */ -static PyObject * -cursor_set_range_dup(CursorObject *self, PyObject *args, PyObject *kwds) -{ - struct cursor_set_range_dup { - MDB_val key; - MDB_val value; - } arg = {{0, 0}, {0, 0}}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_set_range_dup, key)}, - {"value", ARG_BUF, OFFSET(cursor_set_range_dup, value)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - self->key = arg.key; - self->val = arg.value; - return _cursor_get(self, MDB_GET_BOTH_RANGE); -} - -/** - * Cursor.value() -> result - */ -static PyObject * -cursor_value(CursorObject *self) -{ - if(! self->valid) { - return err_invalid(); - } - /* Must refresh `key` and `val` following mutation. */ - if(self->last_mutation != self->trans->mutations && - _cursor_get_c(self, MDB_GET_CURRENT)) { - return NULL; - } - return obj_from_val(&self->val, self->trans->flags & TRANS_BUFFERS); -} - -static PyObject * -new_iterator(CursorObject *cursor, IterValFunc val_func, MDB_cursor_op op) -{ - IterObject *iter = PyObject_New(IterObject, &PyIterator_Type); - if(iter) { - iter->val_func = val_func; - iter->curs = cursor; - Py_INCREF(cursor); - iter->started = 0; - iter->op = op; - } - return (PyObject *) iter; -} - -static PyObject * -iter_from_args(CursorObject *self, PyObject *args, PyObject *kwds, - signed int pos_op, enum MDB_cursor_op op, - int keys_default, int values_default) -{ - struct iter_from_args { - int keys; - int values; - } arg = {keys_default, values_default}; - - static const struct argspec argspec[] = { - {"keys", ARG_BOOL, OFFSET(iter_from_args, keys)}, - {"values", ARG_BOOL, OFFSET(iter_from_args, values)} - }; - void *val_func; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - - if(pos_op != -1 && !self->positioned) { - if(_cursor_get_c(self, (enum MDB_cursor_op) pos_op)) { - return NULL; - } - } - - if(! arg.values) { - val_func = cursor_key; - } else if(! arg.keys) { - val_func = cursor_value; - } else { - val_func = cursor_item; - } - return new_iterator(self, val_func, op); -} - -static PyObject * -cursor_iter(CursorObject *self) -{ - return iter_from_args(self, NULL, NULL, MDB_FIRST, MDB_NEXT, 1, 1); -} - -/** - * Cursor.iternext() -> Iterator - */ -static PyObject * -cursor_iternext(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, MDB_FIRST, MDB_NEXT, 1, 1); -} - -/** - * Cursor.iternext_dup() -> Iterator - */ -static PyObject * -cursor_iternext_dup(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, -1, MDB_NEXT_DUP, 0, 1); -} - -/** - * Cursor.iternext_nodup() -> Iterator - */ -static PyObject * -cursor_iternext_nodup(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, MDB_FIRST, MDB_NEXT_NODUP, 1, 0); -} - -/** - * Cursor.iterprev() -> Iterator - */ -static PyObject * -cursor_iterprev(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, MDB_LAST, MDB_PREV, 1, 1); -} - -/** - * Cursor.iterprev_dup() -> Iterator - */ -static PyObject * -cursor_iterprev_dup(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, -1, MDB_PREV_DUP, 0, 1); -} - -/** - * Cursor.iterprev_nodup() -> Iterator - */ -static PyObject * -cursor_iterprev_nodup(CursorObject *self, PyObject *args, PyObject *kwargs) -{ - return iter_from_args(self, args, kwargs, MDB_LAST, MDB_PREV_NODUP, 1, 0); -} - -/** - * Cursor._iter_from() -> Iterator - */ -static PyObject * -cursor_iter_from(CursorObject *self, PyObject *args) -{ - struct cursor_iter_from { - MDB_val key; - int reverse; - } arg = {{0, 0}, 0}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(cursor_iter_from, key)}, - {"reverse", ARG_BOOL, OFFSET(cursor_iter_from, reverse)} - }; - enum MDB_cursor_op op; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, NULL, &arg)) { - return NULL; - } - - if((! arg.key.mv_size) && (! arg.reverse)) { - rc = _cursor_get_c(self, MDB_FIRST); - } else { - self->key = arg.key; - rc = _cursor_get_c(self, MDB_SET_RANGE); - } - - if(rc) { - return NULL; - } - - op = MDB_NEXT; - if(arg.reverse) { - op = MDB_PREV; - if(! self->positioned) { - if(_cursor_get_c(self, MDB_LAST)) { - return NULL; - } - } - } - - DEBUG("positioned? %d", self->positioned) - return new_iterator(self, (void *)cursor_item, op); -} - -/** - * Cursor.__enter__() - */ -static PyObject *cursor_enter(CursorObject *self) -{ - Py_INCREF(self); - return (PyObject *)self; -} - -/** - * Cursor.__exit__() - */ -static PyObject *cursor_exit(CursorObject *self, PyObject *args) -{ - cursor_clear(self); - Py_RETURN_NONE; -} - -/** - * Cursor.close() - */ -static PyObject *cursor_close(CursorObject *self) -{ - cursor_clear(self); - Py_RETURN_NONE; -} - -static struct PyMethodDef cursor_methods[] = { - {"__enter__", (PyCFunction)cursor_enter, METH_NOARGS}, - {"__exit__", (PyCFunction)cursor_exit, METH_VARARGS}, - {"close", (PyCFunction)cursor_close, METH_NOARGS}, - {"count", (PyCFunction)cursor_count, METH_NOARGS}, - {"delete", (PyCFunction)cursor_delete, METH_VARARGS|METH_KEYWORDS}, - {"first", (PyCFunction)cursor_first, METH_NOARGS}, - {"first_dup", (PyCFunction)cursor_first_dup, METH_NOARGS}, - {"get", (PyCFunction)cursor_get, METH_VARARGS|METH_KEYWORDS}, - {"item", (PyCFunction)cursor_item, METH_NOARGS}, - {"iternext", (PyCFunction)cursor_iternext, METH_VARARGS|METH_KEYWORDS}, - {"iternext_dup", (PyCFunction)cursor_iternext_dup, METH_VARARGS|METH_KEYWORDS}, - {"iternext_nodup", (PyCFunction)cursor_iternext_nodup, METH_VARARGS|METH_KEYWORDS}, - {"iterprev", (PyCFunction)cursor_iterprev, METH_VARARGS|METH_KEYWORDS}, - {"iterprev_dup", (PyCFunction)cursor_iterprev_dup, METH_VARARGS|METH_KEYWORDS}, - {"iterprev_nodup", (PyCFunction)cursor_iterprev_nodup, METH_VARARGS|METH_KEYWORDS}, - {"key", (PyCFunction)cursor_key, METH_NOARGS}, - {"last", (PyCFunction)cursor_last, METH_NOARGS}, - {"last_dup", (PyCFunction)cursor_last_dup, METH_NOARGS}, - {"next", (PyCFunction)cursor_next, METH_NOARGS}, - {"next_dup", (PyCFunction)cursor_next_dup, METH_NOARGS}, - {"next_nodup", (PyCFunction)cursor_next_nodup, METH_NOARGS}, - {"prev", (PyCFunction)cursor_prev, METH_NOARGS}, - {"prev_dup", (PyCFunction)cursor_prev_dup, METH_NOARGS}, - {"prev_nodup", (PyCFunction)cursor_prev_nodup, METH_NOARGS}, - {"put", (PyCFunction)cursor_put, METH_VARARGS|METH_KEYWORDS}, - {"putmulti", (PyCFunction)cursor_put_multi, METH_VARARGS|METH_KEYWORDS}, - {"replace", (PyCFunction)cursor_replace, METH_VARARGS|METH_KEYWORDS}, - {"pop", (PyCFunction)cursor_pop, METH_VARARGS|METH_KEYWORDS}, - {"set_key", (PyCFunction)cursor_set_key, METH_O}, - {"set_key_dup", (PyCFunction)cursor_set_key_dup, METH_VARARGS|METH_KEYWORDS}, - {"set_range", (PyCFunction)cursor_set_range, METH_O}, - {"set_range_dup", (PyCFunction)cursor_set_range_dup, METH_VARARGS|METH_KEYWORDS}, - {"value", (PyCFunction)cursor_value, METH_NOARGS}, - {"_iter_from", (PyCFunction)cursor_iter_from, METH_VARARGS}, - {NULL, NULL} -}; - -static PyTypeObject PyCursor_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "Cursor", /*tp_name*/ - sizeof(CursorObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor) cursor_dealloc,/*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - (inquiry) cursor_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - (getiterfunc)cursor_iter, /*tp_iter*/ - 0, /*tp_iternext*/ - cursor_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - cursor_new, /*tp_new*/ - 0, /*tp_free*/ -}; - - -/* --------- */ -/* Iterators */ -/* --------- */ - - -/** - * Iterator.__del__() - */ -static void -iter_dealloc(IterObject *self) -{ - DEBUG("destroying iterator") - Py_CLEAR(self->curs); - PyObject_Del(self); -} - -/** - * Iterator.__iter__() -> self - */ -static PyObject * -iter_iter(IterObject *self) -{ - Py_INCREF(self); - return (PyObject *)self; -} - -/** - * Iterator.next() -> result - */ -static PyObject * -iter_next(IterObject *self) -{ - if(! self->curs->valid) { - return err_invalid(); - } - if(! self->curs->positioned) { - return NULL; - } - - if(self->started) { - if(_cursor_get_c(self->curs, self->op)) { - return NULL; - } - if(! self->curs->positioned) { - return NULL; - } - } - - self->started = 1; - return self->val_func(self->curs); -} - -static struct PyMethodDef iter_methods[] = { - {"next", (PyCFunction)cursor_next, METH_NOARGS}, - {NULL, NULL} -}; - -static PyTypeObject PyIterator_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "Iterator", /*tp_name*/ - sizeof(IterObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor) iter_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - (getiterfunc)iter_iter, /*tp_iter*/ - (iternextfunc)iter_next, /*tp_iternext*/ - iter_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - 0, /*tp_new*/ - 0, /*tp_free*/ -}; - - - -/* ------------ */ -/* Transactions */ -/* ------------ */ - -static int -trans_clear(TransObject *self) -{ - INVALIDATE(self) -#ifdef HAVE_MEMSINK - ms_notify((PyObject *) self, &self->sink_head); -#endif - - if(self->txn) { - Py_BEGIN_ALLOW_THREADS - MDEBUG("aborting") - mdb_txn_abort(self->txn); - Py_END_ALLOW_THREADS - self->txn = NULL; - } - MDEBUG("db is/was %p", self->db) - Py_CLEAR(self->db); - self->valid = 0; - if(self->env) { - UNLINK_CHILD(self->env, self) - Py_CLEAR(self->env); - } - return 0; -} - -/** - * Transaction.__del__() - */ -static void -trans_dealloc(TransObject *self) -{ - if(self->weaklist != NULL) { - MDEBUG("Clearing weaklist..") - PyObject_ClearWeakRefs((PyObject *) self); - } - - if(self->env && self->txn && - (self->env->max_spare_txns > 0) && (self->flags & TRANS_RDONLY)) { - MDEBUG("caching trans") - if(! (self->flags & TRANS_SPARE)) { - MDEBUG("resetting") - mdb_txn_reset(self->txn); - self->flags |= TRANS_SPARE; - } - self->spare_next = self->env->spare_txns; - self->env->spare_txns = self; - self->env->max_spare_txns--; - - Py_CLEAR(self->db); - UNLINK_CHILD(self->env, self) - Py_CLEAR(self->env); - } else { - MDEBUG("deleting trans") - trans_clear(self); - PyObject_Del(self); - } -} - -/** - * Transaction(env, db) -> new instance. - */ -static PyObject * -trans_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - struct trans_new { - EnvObject *env; - DbObject *db; - TransObject *parent; - int write; - int buffers; - } arg = {NULL, NULL, NULL, 0, 0}; - - static const struct argspec argspec[] = { - {"env", ARG_ENV, OFFSET(trans_new, env)}, - {"db", ARG_DB, OFFSET(trans_new, db)}, - {"parent", ARG_TRANS, OFFSET(trans_new, parent)}, - {"write", ARG_BOOL, OFFSET(trans_new, write)}, - {"buffers", ARG_BOOL, OFFSET(trans_new, buffers)} - }; - - static PyObject *cache = NULL; - if(parse_args(1, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! arg.env) { - return type_error("'env' argument required"); - } - return make_trans(arg.env, arg.db, arg.parent, arg.write, arg.buffers); -} - -/** - * Transaction.abort() - */ -static PyObject * -trans_abort(TransObject *self) -{ - if(self->valid) { - DEBUG("invalidate") - INVALIDATE(self) -#ifdef HAVE_MEMSINK - ms_notify((PyObject *) self, &self->sink_head); -#endif - if(self->flags & TRANS_RDONLY) { - DEBUG("resetting") - /* Reset to spare state, ready for _dealloc to freelist it. */ - mdb_txn_reset(self->txn); - self->flags |= TRANS_SPARE; - } else { - DEBUG("aborting") - Py_BEGIN_ALLOW_THREADS - mdb_txn_abort(self->txn); - Py_END_ALLOW_THREADS - self->txn = NULL; - } - self->valid = 0; - } - Py_RETURN_NONE; -} - -/** - * Transaction.commit() - */ -static PyObject * -trans_commit(TransObject *self) -{ - int rc; - - if(! self->valid) { - return err_invalid(); - } - DEBUG("invalidate") - INVALIDATE(self) -#ifdef HAVE_MEMSINK - ms_notify((PyObject *) self, &self->sink_head); -#endif - if(self->flags & TRANS_RDONLY) { - DEBUG("resetting") - /* Reset to spare state, ready for _dealloc to freelist it. */ - mdb_txn_reset(self->txn); - self->flags |= TRANS_SPARE; - } else { - DEBUG("committing") - UNLOCKED(rc, mdb_txn_commit(self->txn)); - self->txn = NULL; - if(rc) { - return err_set("mdb_txn_commit", rc); - } - } - self->valid = 0; - Py_RETURN_NONE; -} - -/** - * Transaction.cursor() -> Cursor - */ -static PyObject * -trans_cursor(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_cursor { - DbObject *db; - } arg = {self->db}; - - static const struct argspec argspec[] = { - {"db", ARG_DB, OFFSET(trans_cursor, db)} - }; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - return make_cursor(arg.db, self); -} - -/** - * Transaction.delete() -> bool - */ -static PyObject * -trans_delete(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_delete { - MDB_val key; - MDB_val val; - DbObject *db; - } arg = {{0, 0}, {0, 0}, self->db}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(trans_delete, key)}, - {"value", ARG_BUF, OFFSET(trans_delete, val)}, - {"db", ARG_DB, OFFSET(trans_delete, db)} - }; - MDB_val *val_ptr; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - val_ptr = arg.val.mv_size ? &arg.val : NULL; - self->mutations++; - UNLOCKED(rc, mdb_del(self->txn, arg.db->dbi, &arg.key, val_ptr)); - if(rc) { - if(rc == MDB_NOTFOUND) { - Py_RETURN_FALSE; - } - return err_set("mdb_del", rc); - } - Py_RETURN_TRUE; -} - -/** - * Transaction.drop(db) - */ -static PyObject * -trans_drop(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_drop { - DbObject *db; - int delete; - } arg = {NULL, 1}; - - static const struct argspec argspec[] = { - {"db", ARG_DB, OFFSET(trans_drop, db)}, - {"delete", ARG_BOOL, OFFSET(trans_drop, delete)} - }; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! arg.db) { - return type_error("'db' argument required."); - } else if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - UNLOCKED(rc, mdb_drop(self->txn, arg.db->dbi, arg.delete)); - self->mutations++; - if(rc) { - return err_set("mdb_drop", rc); - } - Py_RETURN_NONE; -} - -/** - * Transaction.get() -> result - */ -static PyObject * -trans_get(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_get { - MDB_val key; - PyObject *default_; - DbObject *db; - } arg = {{0, 0}, Py_None, self->db}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(trans_get, key)}, - {"default", ARG_OBJ, OFFSET(trans_get, default_)}, - {"db", ARG_DB, OFFSET(trans_get, db)} - }; - MDB_val val; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - if(! arg.key.mv_data) { - return type_error("key must be given."); - } - - Py_BEGIN_ALLOW_THREADS - rc = mdb_get(self->txn, arg.db->dbi, &arg.key, &val); - preload(rc, val.mv_data, val.mv_size); - Py_END_ALLOW_THREADS - - if(rc) { - if(rc == MDB_NOTFOUND) { - Py_INCREF(arg.default_); - return arg.default_; - } - return err_set("mdb_get", rc); - } - return obj_from_val(&val, self->flags & TRANS_BUFFERS); -} - -/** - * Transaction.put() -> bool - */ -static PyObject * -trans_put(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_put { - MDB_val key; - MDB_val value; - int dupdata; - int overwrite; - int append; - DbObject *db; - } arg = {{0, 0}, {0, 0}, 1, 1, 0, self->db}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(trans_put, key)}, - {"value", ARG_BUF, OFFSET(trans_put, value)}, - {"dupdata", ARG_BOOL, OFFSET(trans_put, dupdata)}, - {"overwrite", ARG_BOOL, OFFSET(trans_put, overwrite)}, - {"append", ARG_BOOL, OFFSET(trans_put, append)}, - {"db", ARG_DB, OFFSET(trans_put, db)} - }; - int flags; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - flags = 0; - if(! arg.dupdata) { - flags |= MDB_NODUPDATA; - } - if(! arg.overwrite) { - flags |= MDB_NOOVERWRITE; - } - if(arg.append) { - flags |= MDB_APPEND; - } - - DEBUG("inserting '%.*s' (%d) -> '%.*s' (%d)", - (int)arg.key.mv_size, (char *)arg.key.mv_data, - (int)arg.key.mv_size, - (int)arg.value.mv_size, (char *)arg.value.mv_data, - (int)arg.value.mv_size) - - self->mutations++; - UNLOCKED(rc, mdb_put(self->txn, (arg.db)->dbi, - &arg.key, &arg.value, flags)); - if(rc) { - if(rc == MDB_KEYEXIST) { - Py_RETURN_FALSE; - } - return err_set("mdb_put", rc); - } - Py_RETURN_TRUE; -} - -static PyObject * -make_cursor(DbObject *db, TransObject *trans); -static PyObject * -do_cursor_replace(CursorObject *self, MDB_val *key, MDB_val *val); - -/** - * Transaction.replace() -> None|result - */ -static PyObject * -trans_replace(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_replace { - MDB_val key; - MDB_val value; - DbObject *db; - } arg = {{0, 0}, {0, 0}, self->db}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(trans_replace, key)}, - {"value", ARG_BUF, OFFSET(trans_replace, value)}, - {"db", ARG_DB, OFFSET(trans_replace, db)} - }; - PyObject *ret; - CursorObject *cursor; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - ret = NULL; - cursor = (CursorObject *) make_cursor(arg.db, self); - if(cursor) { - ret = do_cursor_replace(cursor, &arg.key, &arg.value); - Py_DECREF(cursor); - } - return ret; -} - -static int -_cursor_get_c(CursorObject *self, enum MDB_cursor_op op); - -/** - * Transaction.pop() -> None|result - */ -static PyObject * -trans_pop(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_pop { - MDB_val key; - DbObject *db; - } arg = {{0, 0}, self->db}; - - static const struct argspec argspec[] = { - {"key", ARG_BUF, OFFSET(trans_pop, key)}, - {"db", ARG_DB, OFFSET(trans_pop, db)} - }; - CursorObject *cursor; - PyObject *old; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - if(! ((cursor = (CursorObject *) make_cursor(arg.db, self)))) { - return NULL; - } - - cursor->key = arg.key; - if(_cursor_get_c(cursor, MDB_SET_KEY)) { - Py_DECREF((PyObject *)cursor); - return NULL; - } - if(! cursor->positioned) { - Py_DECREF((PyObject *)cursor); - Py_RETURN_NONE; - } - if(! ((old = obj_from_val(&cursor->val, 0)))) { - Py_DECREF((PyObject *)cursor); - return NULL; - } - - UNLOCKED(rc, mdb_cursor_del(cursor->curs, 0)); - Py_DECREF((PyObject *)cursor); - self->mutations++; - if(rc) { - Py_DECREF(old); - return err_set("mdb_cursor_del", rc); - } - return old; -} - -/** - * Transaction.__enter__() - */ -static PyObject *trans_enter(TransObject *self) -{ - if(! self->valid) { - return err_invalid(); - } - Py_INCREF(self); - return (PyObject *)self; -} - -/** - * Transaction.__exit__() - */ -static PyObject *trans_exit(TransObject *self, PyObject *args) -{ - if(! self->valid) { - return err_invalid(); - } - if(PyTuple_GET_ITEM(args, 0) == Py_None) { - return trans_commit(self); - } else { - return trans_abort(self); - } -} - -/** - * Transaction.stat() -> dict - */ -static PyObject * -trans_stat(TransObject *self, PyObject *args, PyObject *kwds) -{ - struct trans_stat { - DbObject *db; - } arg = {self->db}; - - static const struct argspec argspec[] = { - {"db", ARG_DB, OFFSET(trans_stat, db)} - }; - MDB_stat st; - int rc; - - static PyObject *cache = NULL; - if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { - return NULL; - } - if(! db_owner_check(arg.db, self->env)) { - return NULL; - } - - UNLOCKED(rc, mdb_stat(self->txn, arg.db->dbi, &st)); - if(rc) { - return err_set("mdb_stat", rc); - } - return dict_from_fields(&st, mdb_stat_fields); -} - -static struct PyMethodDef trans_methods[] = { - {"__enter__", (PyCFunction)trans_enter, METH_NOARGS}, - {"__exit__", (PyCFunction)trans_exit, METH_VARARGS}, - {"abort", (PyCFunction)trans_abort, METH_NOARGS}, - {"commit", (PyCFunction)trans_commit, METH_NOARGS}, - {"cursor", (PyCFunction)trans_cursor, METH_VARARGS|METH_KEYWORDS}, - {"delete", (PyCFunction)trans_delete, METH_VARARGS|METH_KEYWORDS}, - {"drop", (PyCFunction)trans_drop, METH_VARARGS|METH_KEYWORDS}, - {"get", (PyCFunction)trans_get, METH_VARARGS|METH_KEYWORDS}, - {"put", (PyCFunction)trans_put, METH_VARARGS|METH_KEYWORDS}, - {"replace", (PyCFunction)trans_replace, METH_VARARGS|METH_KEYWORDS}, - {"pop", (PyCFunction)trans_pop, METH_VARARGS|METH_KEYWORDS}, - {"stat", (PyCFunction)trans_stat, METH_VARARGS|METH_KEYWORDS}, - {NULL, NULL} -}; - -static PyTypeObject PyTransaction_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "Transaction", /*tp_name*/ - sizeof(TransObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor) trans_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - (inquiry) trans_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(TransObject, weaklist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - trans_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - trans_new, /*tp_new*/ - 0, /*tp_free*/ -}; - - -/** - * lmdb.enable_drop_gil() - */ -static PyObject * -enable_drop_gil(void) -{ - Py_RETURN_NONE; -} - -/** - * lmdb.get_version() -> tuple - */ -static PyObject * -get_version(void) -{ - return Py_BuildValue("iii", MDB_VERSION_MAJOR, - MDB_VERSION_MINOR, MDB_VERSION_PATCH); -} - -static struct PyMethodDef module_methods[] = { - {"enable_drop_gil", (PyCFunction) enable_drop_gil, METH_NOARGS, ""}, - {"version", (PyCFunction) get_version, METH_NOARGS, ""}, - {0, 0, 0, 0} -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "cpython", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL -}; -#endif - -/** - * Initialize and publish the LMDB built-in types. - */ -static int init_types(PyObject *mod) -{ - static PyTypeObject *types[] = { - &PyEnvironment_Type, - &PyCursor_Type, - &PyTransaction_Type, - &PyIterator_Type, - &PyDatabase_Type, - NULL - }; - - int i; - for(i = 0; types[i]; i++) { - PyTypeObject *type = types[i]; - if(PyType_Ready(type)) { - return -1; - } - if(PyObject_SetAttrString(mod, type->tp_name, (PyObject *)type)) { - return -1; - } - } - return 0; -} - -/** - * Initialize a bunch of constants used to ease number compares. - */ -static int init_constants(PyObject *mod) -{ - if(! ((py_zero = PyLong_FromUnsignedLongLong(0)))) { - return -1; - } - if(! ((py_int_max = PyLong_FromUnsignedLongLong(INT_MAX)))) { - return -1; - } - if(! ((py_size_max = PyLong_FromUnsignedLongLong(SIZE_MAX)))) { - return -1; - } - return 0; -} - -/** - * Create lmdb.Error exception class, and one subclass for each entry in - * `error_map`. - */ -static int init_errors(PyObject *mod) -{ - size_t count; - char qualname[64]; - int i; - - Error = PyErr_NewException("lmdb.Error", NULL, NULL); - if(! Error) { - return -1; - } - if(PyObject_SetAttrString(mod, "Error", Error)) { - return -1; - } - - count = (sizeof error_map / sizeof error_map[0]); - error_tbl = malloc(sizeof(PyObject *) * count); - if(! error_tbl) { - return -1; - } - - for(i = 0; i < count; i++) { - const struct error_map *error = &error_map[i]; - PyObject *klass; - - snprintf(qualname, sizeof qualname, "lmdb.%s", error->name); - qualname[sizeof qualname - 1] = '\0'; - - if(! ((klass = PyErr_NewException(qualname, Error, NULL)))) { - return -1; - } - - error_tbl[i] = klass; - if(PyObject_SetAttrString(mod, error->name, klass)) { - return -1; - } - } - return 0; -} - -/** - * Do all required to initialize the lmdb.cpython module. - */ -PyMODINIT_FUNC -MODINIT_NAME(void) -{ -#if PY_MAJOR_VERSION >= 3 - PyObject *mod = PyModule_Create(&moduledef); -#else - PyObject *mod = Py_InitModule3("cpython", module_methods, ""); -#endif - if(! mod) { - MOD_RETURN(NULL); - } - - if(init_types(mod)) { - MOD_RETURN(NULL); - } - -#ifdef HAVE_MEMSINK - MemSink_IMPORT; - if(ms_init_source(&PyTransaction_Type, offsetof(TransObject, sink_head))) { - MOD_RETURN(NULL); - } -#endif - -#ifdef _WIN32 - if(! ((msvcrt = PyImport_ImportModule("msvcrt")))) { - MOD_RETURN(NULL); - } -#endif - - if(init_constants(mod)) { - MOD_RETURN(NULL); - } - if(init_errors(mod)) { - MOD_RETURN(NULL); - } - if(PyObject_SetAttrString(mod, "open", (PyObject *)&PyEnvironment_Type)) { - MOD_RETURN(NULL); - } - MOD_RETURN(mod); -} diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/__init__.py stb-tester-31/vendor/py-lmdb/lmdb/__init__.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/__init__.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lmdb/__init__.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . - -""" -cffi wrapper for OpenLDAP's "Lightning" MDB database. - -Please see http://lmdb.readthedocs.org/ -""" - -import os -import sys - -def _reading_docs(): - # Hack: disable speedups while testing or reading docstrings. - basename = os.path.basename(sys.argv[0]) - return any(x in basename for x in ('sphinx-build', 'pydoc')) - -try: - if _reading_docs() or os.getenv('LMDB_FORCE_CFFI') is not None: - raise ImportError - from lmdb.cpython import * -except ImportError: - from lmdb.cffi import * - from lmdb.cffi import __all__ - from lmdb.cffi import __doc__ - -__version__ = '0.86' - -# Hack to support Python v2.5 'python -mlmdb' -if __name__ == '__main__': - import lmdb.tool - import atexit - atexit.register(lmdb.tool.main) diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/__main__.py stb-tester-31/vendor/py-lmdb/lmdb/__main__.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/__main__.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lmdb/__main__.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,24 +0,0 @@ -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . - -# Hack to support Python >=v2.6 'pythom -mlmdb' -from __future__ import absolute_import -import lmdb.tool -lmdb.tool.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/tool.py stb-tester-31/vendor/py-lmdb/lmdb/tool.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/lmdb/tool.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/lmdb/tool.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,610 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -""" -Basic tools for working with LMDB. - - copy: Consistent high speed backup an environment. - %prog copy -e source.lmdb target.lmdb - - copyfd: Consistent high speed backup an environment to stdout. - %prog copyfd -e source.lmdb > target.lmdb/data.mdb - - drop: Delete one or more sub-databases. - %prog drop db1 - - dump: Dump one or more databases to disk in 'cdbmake' format. - Usage: dump [db1=file1.cdbmake db2=file2.cdbmake] - - If no databases are given, dumps the main database to 'main.cdbmake'. - - edit: Add/delete/replace values from a database. - %prog edit --set key=value --set-file key=/path \\ - --add key=value --add-file key=/path/to/file \\ - --delete key - - get: Read one or more values from a database. - %prog get [ [ [..]]] - - readers: Display readers in the lock table - %prog readers -e /path/to/db [-c] - - If -c is specified, clear stale readers. - - restore: Read one or more database from disk in 'cdbmake' format. - %prog restore db1=file1.cdbmake db2=file2.cdbmake - - The special db name ":main:" may be used to indicate the main DB. - - rewrite: Re-create an environment using MDB_APPEND - %prog rewrite -e src.lmdb -E dst.lmdb [ [ ..]] - - If no databases are given, rewrites only the main database. - - shell: Open interactive console with ENV set to the open environment. - - stat: Print environment statistics. - - warm: Read environment into page cache sequentially. - - watch: Show live environment statistics -""" - -from __future__ import absolute_import -from __future__ import with_statement -import array -import collections -import contextlib -import csv -import functools -import optparse -import os -import pprint -import signal -import string -import struct -import sys -import time - -# Python3.x bikeshedded trechery. -try: - from io import BytesIO as StringIO -except ImportError: - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - -import lmdb - - -BUF_SIZE = 10485760 -ENV = None -DB = None - - -def isprint(c): - """Return ``True`` if the character `c` can be printed visibly and without - adversely affecting printing position (e.g. newline).""" - return c in string.printable and ord(c) > 16 - - -def xxd(s): - """Return a vaguely /usr/bin/xxd formatted representation of the bytestring - `s`.""" - sio = StringIO() - for idx, ch in enumerate(s): - if not (idx % 16): - if idx: - sio.write(' ') - sio.write(pr) - sio.write('\n') - sio.write('%07x:' % idx) - pr = '' - if not (idx % 2): - sio.write(' ') - sio.write('%02x' % (ord(ch),)) - pr += ch if isprint(ch) else '.' - - if idx % 16: - need = 15 - (idx % 16) - # fill remainder of last line. - sio.write(' ' * need) - sio.write(' ' * (need / 2)) - sio.write(' ') - sio.write(pr) - - sio.write('\n') - return sio.getvalue() - - -def make_parser(): - parser = optparse.OptionParser() - parser.prog = 'python -mlmdb' - parser.usage = '%prog [options] \n' + __doc__.rstrip() - parser.add_option('-e', '--env', help='Environment file to open') - parser.add_option('-d', '--db', help='Database to open (default: main)') - parser.add_option('-r', '--read', help='Open environment read-only') - parser.add_option('-S', '--map_size', type='int', default='10', - help='Map size in megabytes (default: 10)') - parser.add_option('-a', '--all', action='store_true', - help='Make "dump" dump all databases') - parser.add_option('-T', '--txn_size', type='int', default=1000, - help='Writes per transaction (default: 1000)') - parser.add_option('-E', '--target_env', - help='Target environment file for "dumpfd"') - parser.add_option('-x', '--xxd', action='store_true', - help='Print values in xxd format') - parser.add_option('-M', '--max-dbs', type='int', default=128, - help='Maximum open DBs (default: 128)') - parser.add_option('--out-fd', type='int', default=1, - help='"copyfd" command target fd') - group = parser.add_option_group('Options for "edit" command') - group.add_option('--set', action='append', - help='List of key=value pairs to set.') - group.add_option('--set-file', action='append', - help='List of key pairs to read from files.') - group.add_option('--add', action='append', - help='List of key=value pairs to add.') - group.add_option('--add-file', action='append', - help='List of key pairs to read from files.') - group.add_option('--delete', action='append', - help='List of key=value pairs to delete.') - group = parser.add_option_group('Options for "readers" command') - group.add_option('-c', '--clean', action='store_true', - help='Clean stale readers? (default: no)') - group = parser.add_option_group('Options for "watch" command') - group.add_option('--csv', action='store_true', - help='Generate CSV instead of terminal output.') - group.add_option('--interval', type='int', default=1, - help='Interval size (default: 1sec)') - group.add_option('--window', type='int', default=10, - help='Average window size (default: 10)') - return parser - - -def die(fmt, *args): - if args: - fmt %= args - sys.stderr.write('lmdb.tool: %s\n' % (fmt,)) - raise SystemExit(1) - - -def dump_cursor_to_fp(cursor, fp): - for key, value in cursor: - fp.write('+%d,%d:' % (len(key), len(value))) - fp.write(key) - fp.write('->') - fp.write(value) - fp.write('\n') - fp.write('\n') - - -def db_map_from_args(args): - db_map = {} - - for arg in args: - dbname, sep, path = arg.partition('=') - if not sep: - die('DB specification missing "=": %r', arg) - - if dbname == ':main:': - dbname = None - if dbname in db_map: - die('DB specified twice: %r', arg) - db_map[dbname] = (ENV.open_db(dbname), path) - - if not db_map: - db_map[':main:'] = (ENV.open_db(None), 'main.cdbmake') - return db_map - - -def cmd_copy(opts, args): - if len(args) != 1: - die('Please specify output directory (see --help)') - - output_dir = args[0] - if os.path.exists(output_dir): - die('Output directory %r already exists.', output_dir) - - os.makedirs(output_dir, int('0755', 8)) - print('Running copy to %r....' % (output_dir,)) - ENV.copy(output_dir) - - -def cmd_copyfd(opts, args): - if args: - die('"copyfd" command takes no arguments (see --help)') - - try: - fp = os.fdopen(opts.out_fd, 'w', 0) - except OSError: - e = sys.exc_info()[1] - die('Bad --out-fd %d: %s', opts.out_fd, e) - - ENV.copyfd(opts.out_fd) - - -def cmd_dump(opts, args): - db_map = db_map_from_args(args) - with ENV.begin(buffers=True) as txn: - for dbname, (db, path) in db_map.iteritems(): - with open(path, 'wb', BUF_SIZE) as fp: - print('Dumping to %r...' % (path,)) - cursor = txn.cursor(db=db) - dump_cursor_to_fp(cursor, fp) - - -def restore_cursor_from_fp(txn, fp, db): - read = fp.read - read1 = functools.partial(read, 1) - read_until = lambda sep: ''.join(iter(read1, sep)) - - rec_nr = 0 - - while True: - rec_nr += 1 - plus = read(1) - if plus == '\n': - break - elif plus != '+': - die('bad or missing plus, line/record #%d', rec_nr) - - bad = False - try: - klen = int(read_until(','), 10) - dlen = int(read_until(':'), 10) - except ValueError: - die('bad or missing length, line/record #%d', rec_nr) - - key = read(klen) - if read(2) != '->': - die('bad or missing separator, line/record #%d', rec_nr) - - data = read(dlen) - if (len(key) + len(data)) != (klen + dlen): - die('short key or data, line/record #%d', rec_nr) - - if read(1) != '\n': - die('bad line ending, line/record #%d', rec_nr) - - txn.put(key, data, db=db) - - return rec_nr - - -def cmd_drop(opts, args): - if not args: - die('Must specify at least one sub-database (see --help)') - - dbs = map(ENV.open_db, args) - for idx, db in enumerate(dbs): - name = args[idx] - if name == ':main:': - die('Cannot drop main DB') - print('Dropping DB %r...' % (name,)) - with ENV.begin(write=True) as txn: - txn.drop(db) - - -def cmd_readers(opts, args): - if opts.clean: - print('Cleaned %d stale entries.' % (ENV.reader_check(),)) - print(ENV.readers()) - - -def cmd_restore(opts, args): - db_map = db_map_from_args(args) - with ENV.begin(buffers=True, write=True) as txn: - for dbname, (db, path) in db_map.iteritems(): - with open(path, 'rb', BUF_SIZE) as fp: - print('Restoring from %r...' % (path,)) - count = restore_cursor_from_fp(txn, fp, db) - print('Loaded %d keys from %r' % (count, path)) - - -def delta(hst): - return [(hst[i] - hst[i-1]) for i in xrange(1, len(hst))] - - -SYS_BLOCK = '/sys/block' - -def _find_diskstat(path): - if not os.path.exists(SYS_BLOCK): - return - st = os.stat(path) - devs = '%s:%s' % (st.st_dev >> 8, st.st_dev & 0xff) - - def maybe(rootpath): - dpath = os.path.join(rootpath, 'dev') - if os.path.exists(dpath): - with file(dpath) as fp: - if fp.read().strip() == devs: - return os.path.join(rootpath, 'stat') - - for name in os.listdir(SYS_BLOCK): - basepath = os.path.join(SYS_BLOCK, name) - statpath = maybe(basepath) - if statpath: - return statpath - for name in os.listdir(basepath): - base2path = os.path.join(basepath, name) - statpath = maybe(base2path) - if statpath: - return statpath - -class DiskStatter(object): - FIELDS = ( - 'reads', - 'reads_merged', - 'sectors_read', - 'read_ms', - 'writes', - 'writes_merged', - 'sectors_written', - 'write_ms', - 'io_count', - 'io_ms', - 'total_ms' - ) - - def __init__(self, path): - self.fp = file(path) - self.refresh() - - def refresh(self): - self.fp.seek(0) - vars(self).update((self.FIELDS[i], int(s)) - for i, s in enumerate(self.fp.read().split())) - - -def cmd_watch(opts, args): - info = None - stat = None - - def window(func): - history = collections.deque() - def windowfunc(): - history.append(func()) - if len(history) > opts.window: - history.popleft() - if len(history) <= 1: - return 0 - n = sum(delta(history)) / float(len(history) - 1) - return n / opts.interval - return windowfunc - - envmb = lambda: (info['last_pgno'] * stat['psize']) / 1048576. - - cols = [ - ('%d', 'Depth', lambda: stat['depth']), - ('%d', 'Branch', lambda: stat['branch_pages']), - ('%d', 'Leaf', lambda: stat['leaf_pages']), - ('%+d', 'Leaf/s', window(lambda: stat['leaf_pages'])), - ('%d', 'Oflow', lambda: stat['overflow_pages']), - ('%+d', 'Oflow/s', window(lambda: stat['overflow_pages'])), - ('%d', 'Recs', lambda: stat['entries']), - ('%+d', 'Recs/s', window(lambda: stat['entries'])), - ('%d', 'Rdrs', lambda: info['num_readers']), - ('%.2f', 'EnvMb', envmb), - ('%+.2f', 'EnvMb/s', window(envmb)), - ('%d', 'Txs', lambda: info['last_txnid']), - ('%+.2f', 'Txs/s', window(lambda: info['last_txnid'])) - ] - - statter = None - statpath = _find_diskstat(ENV.path()) - if statpath: - statter = DiskStatter(statpath) - cols += [ - #('%d', 'SctRd', lambda: statter.sectors_read), - ('%+d', 'SctRd/s', window(lambda: statter.sectors_read)), - #('%d', 'SctWr', lambda: statter.sectors_written), - ('%+d', 'SctWr/s', window(lambda: statter.sectors_written)), - ] - - term_width = 0 - widths = [len(head) for _, head, _ in cols] - - if opts.csv: - writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - writer.writerow([head for _, head, _ in cols]) - - cnt = 0 - try: - while True: - stat = ENV.stat() - info = ENV.info() - if statter: - statter.refresh() - - vals = [] - for i, (fmt, head, func) in enumerate(cols): - val = fmt % func() - vals.append(val) - widths[i] = max(widths[i], len(val)) - - if opts.csv: - writer.writerow(vals) - else: - if term_width != _TERM_WIDTH or not (cnt % (_TERM_HEIGHT - 2)): - for i, (fmt, head, func) in enumerate(cols): - sys.stdout.write(head.rjust(widths[i] + 1)) - sys.stdout.write('\n') - term_width = _TERM_WIDTH - for i, val in enumerate(vals): - sys.stdout.write(val.rjust(widths[i] + 1)) - sys.stdout.write('\n') - - time.sleep(opts.interval) - cnt += 1 - except KeyboardInterrupt: - pass - - -def cmd_warm(opts, args): - stat = ENV.stat() - info = ENV.info() - - bufsize = 32768 - last_offset = stat['psize'] * info['last_pgno'] - buf = array.array('c', '\x00' * bufsize) - t0 = time.time() - - fp = open(opts.env + '/data.mdb', 'rb', bufsize) - while fp.tell() < last_offset: - fp.readinto(buf) - print('Warmed %.2fmb in %dms' % - (last_offset / 1048576., 1000 * (time.time() - t0))) - - -def cmd_rewrite(opts, args): - if not opts.target_env: - die('Must specify target environment path with -E') - - src_info = ENV.info() - target_env = lmdb.open(opts.target_env, - map_size=src_info['map_size'] * 2, - max_dbs=opts.max_dbs, sync=False, - writemap=True, map_async=True, - metasync=False) - - dbs = [] - for arg in args: - name = None if arg == ':main:' else arg - src_db = ENV.open_db(name) - dst_db = ENV.open_db(name) - dbs.append((arg, src_db, dst_db)) - - if not dbs: - dbs.append((':main:', ENV.open_db(None), target_env.open_db(None))) - - for name, src_db, dst_db in dbs: - print('Writing %r...' % (name,)) - with target_env.begin(db=dst_db, write=True) as wtxn: - for key, value in ENV.cursor(db=src_db, buffers=True): - wtxn.put(key, value, append=True) - - print('Syncing..') - target_env.sync(True) - - -def cmd_get(opts, args): - print_header = len(args) > 1 - - with ENV.begin(buffers=True, db=DB) as txn: - for arg in args: - value = txn.get(arg) - if value is None: - print('%r: missing' % (arg,)) - continue - if print_header: - print('%r:' % (arg,)) - if opts.xxd: - print(xxd(value)) - else: - print(value) - - -def cmd_edit(opts, args): - if args: - die('Edit command only takes options, not arguments (see --help)') - - with ENV.begin(write=True) as txn: - cursor = txn.cursor(db=DB) - for elem in opts.add or []: - key, _, value = elem.partition('=') - cursor.put(key, value, overwrite=False) - - for elem in opts.set or []: - key, _, value = elem.partition('=') - cursor.put(key, value) - - for key in opts.delete or []: - cursor.delete(key) - - for elem in opts.add_file or []: - key, _, path = elem.partition('=') - with open(path, 'rb') as fp: - cursor.put(key, fp.read(), overwrite=False) - - for elem in opts.set_file or []: - key, _, path = elem.partition('=') - with open(path, 'rb') as fp: - cursor.put(key, fp.read()) - - -def cmd_shell(opts, args): - import code - import readline - code.InteractiveConsole(globals()).interact() - - -def cmd_stat(opts, args): - pprint.pprint(ENV.stat()) - pprint.pprint(ENV.info()) - - -def _get_term_width(default=(80, 25)): - try: - import fcntl # No fcntl on win32 - import termios # No termios on win32 - s = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, '1234') - height, width = struct.unpack('hh', s) - return width, height - except: - return default - -def _on_sigwinch(*args): - global _TERM_WIDTH, _TERM_HEIGHT - _TERM_WIDTH, _TERM_HEIGHT = _get_term_width() - -def main(): - parser = make_parser() - opts, args = parser.parse_args() - - if not args: - die('Please specify a command (see --help)') - if not opts.env: - die('Please specify environment (--env)') - - global ENV - ENV = lmdb.open(opts.env, map_size=opts.map_size*1048576, - max_dbs=opts.max_dbs, create=False) - - if opts.db: - global DB - DB = ENV.open_db(opts.db) - - if hasattr(signal, 'SIGWINCH'): # Disable on win32. - signal.signal(signal.SIGWINCH, _on_sigwinch) - _on_sigwinch() - - func = globals().get('cmd_' + args[0]) - if not func: - die('No such command: %r' % (args[0],)) - - func(opts, args[1:]) - - -if __name__ == '__main__': - main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/MANIFEST.in stb-tester-31/vendor/py-lmdb/MANIFEST.in --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/MANIFEST.in 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/MANIFEST.in 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -graft . -graft lib diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/cursor-del-break.c stb-tester-31/vendor/py-lmdb/misc/cursor-del-break.c --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/cursor-del-break.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/cursor-del-break.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,89 +0,0 @@ -// http://www.openldap.org/its/index.cgi/Software%20Bugs?id=7722 -// gcc -g -I src/py-lmdb/lib -o cursor-del-break cursor-del-break.c -#include -#include -#include -#include -#include - -#include "lmdb.h" -#include "mdb.c" -#include "midl.c" - - -void check(int x) -{ - if(x) { - fprintf(stderr, "eek %s\n", mdb_strerror(x)); - _exit(1); - } -} - -#define RECS 2048 -#define DB_PATH "/ram/tdb" - -MDB_dbi dbi; -MDB_txn *txn; -MDB_env *env; -MDB_cursor *c1; - -char recpattern[256]; -MDB_val keyv; -MDB_val valv; - -void new_txn(void) -{ - if(txn) { - fprintf(stderr, "commit\n"); - check(mdb_txn_commit(txn)); - } - check(mdb_txn_begin(env, NULL, 0, &txn)); -} - -int main(void) -{ - check(mdb_env_create(&env)); - check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL)); - check(mdb_env_set_maxreaders(env, 126)); - check(mdb_env_set_maxdbs(env, 1)); - if(! access(DB_PATH, X_OK)) { - system("rm -rf " DB_PATH); - } - check(mkdir(DB_PATH, 0777)); - check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644)); - new_txn(); - check(mdb_dbi_open(txn, NULL, 0, &dbi)); - - // make pattern - int i; - for(i = 0; i < sizeof recpattern; i++) { - recpattern[i] = i % 256; - } - - for(i = 0; i < RECS; i++) { - char keybuf[40]; - keyv.mv_size = sprintf(keybuf, "%08x", i); - keyv.mv_data = keybuf; - valv.mv_size = sizeof recpattern; - valv.mv_data = recpattern; - check(mdb_put(txn, dbi, &keyv, &valv, 0)); - } - - new_txn(); - - check(mdb_cursor_open(txn, dbi, &c1)); - check(mdb_cursor_get(c1, &keyv, &valv, MDB_FIRST)); - check(mdb_del(txn, dbi, &keyv, NULL)); - - for(i = 1; i < RECS; i++) { - check(mdb_cursor_get(c1, &keyv, &valv, MDB_NEXT)); - char keybuf[40]; - int sz = sprintf(keybuf, "%08x", i); - check((!(sz==keyv.mv_size)) || memcmp(keyv.mv_data, keybuf, sz)); - check(memcmp(valv.mv_data, recpattern, sizeof recpattern)); - printf("%d\n", i); - check(mdb_del(txn, dbi, &keyv, NULL)); - } - - new_txn(); -} diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/cursor_put_pyparse.diff stb-tester-31/vendor/py-lmdb/misc/cursor_put_pyparse.diff --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/cursor_put_pyparse.diff 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/cursor_put_pyparse.diff 1970-01-01 00:00:00.000000000 +0000 @@ -1,31 +0,0 @@ -diff --git a/lmdb/cpython.c b/lmdb/cpython.c -index dd1c8b9..ced5ea3 100644 ---- a/lmdb/cpython.c -+++ b/lmdb/cpython.c -@@ -2319,11 +2319,25 @@ cursor_put(CursorObject *self, PyObject *args, PyObject *kwds) - {ARG_BOOL, OVERWRITE_S, OFFSET(cursor_put, overwrite)}, - {ARG_BOOL, APPEND_S, OFFSET(cursor_put, append)} - }; -+ static char *keywords[] = { -+ "key", "value", "dupdata", "overwrite", "append", NULL -+ }; -+ PyObject *key, *val; - int flags; - int rc; - - static PyObject *cache = NULL; -- if(parse_args(self->valid, SPECSIZE(), argspec, &cache, args, kwds, &arg)) { -+ if(! self->valid) { -+ return err_invalid(); -+ } -+ if(! PyArg_ParseTupleAndKeywords(args, kwds, "OO|iii", keywords, -+ &key, &val, &arg.dupdata, &arg.overwrite, -+ &arg.append)) { -+ return NULL; -+ } -+ -+ if(val_from_buffer(&arg.key, key) || -+ val_from_buffer(&arg.val, val)) { - return NULL; - } - diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/gdb.commands stb-tester-31/vendor/py-lmdb/misc/gdb.commands --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/gdb.commands 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/gdb.commands 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -# vim:syntax=gdb -# -# Print a backtrace if a program crashes. Run using: -# gdb -x misc/gdb.commands --args argv0 argv1 .. -# - -set confirm off - -define hook-stop - init-if-undefined $_exitcode = 999 - if $_exitcode == 999 - echo Abnormal stop.\n - backtrace - quit 2 - else - echo Normal exit.\n - quit $_exitcode - end -end - -run diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/its7733.c stb-tester-31/vendor/py-lmdb/misc/its7733.c --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/its7733.c 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/its7733.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,77 +0,0 @@ -// http://www.openldap.org/its/index.cgi/Software%20Bugs?id=7733 -// gcc -g -I ../lib -o its7733 its7733.c -#include -#include -#include -#include -#include - -#include "lmdb.h" -#include "mdb.c" -#include "midl.c" - - -void check(int x) -{ - if(x) { - fprintf(stderr, "eek %s\n", mdb_strerror(x)); - _exit(1); - } -} - -#define DB_PATH "/ram/tdb" - -MDB_dbi dbi; -MDB_txn *txn; -MDB_env *env; -MDB_cursor *c1; - -MDB_val keyv; -MDB_val valv; - -void new_txn(void) -{ - if(txn) { - fprintf(stderr, "commit\n"); - check(mdb_txn_commit(txn)); - } - check(mdb_txn_begin(env, NULL, 0, &txn)); -} - - -void put(const char *k) -{ - keyv.mv_size = strlen(k); - keyv.mv_data = k; - valv.mv_size = 0; - valv.mv_data = ""; - check(mdb_put(txn, dbi, &keyv, &valv, 0)); -} - -int main(void) -{ - check(mdb_env_create(&env)); - check(mdb_env_set_mapsize(env, 1048576UL*1024UL*3UL)); - check(mdb_env_set_maxreaders(env, 126)); - check(mdb_env_set_maxdbs(env, 1)); - if(! access(DB_PATH, X_OK)) { - system("rm -rf " DB_PATH); - } - check(mkdir(DB_PATH, 0777)); - check(mdb_env_open(env, DB_PATH, MDB_MAPASYNC|MDB_NOSYNC|MDB_NOMETASYNC, 0644)); - new_txn(); - check(mdb_dbi_open(txn, NULL, 0, &dbi)); - - put("a"); - put("b"); - put("baa"); - put("d"); - - new_txn(); - - check(mdb_cursor_open(txn, dbi, &c1)); - check(mdb_cursor_get(c1, &keyv, &valv, MDB_LAST)); - check(mdb_cursor_del(c1, 0)); - check(mdb_cursor_del(c1, 0)); - new_txn(); -} diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/readers_mrb_env.patch stb-tester-31/vendor/py-lmdb/misc/readers_mrb_env.patch --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/readers_mrb_env.patch 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/readers_mrb_env.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,44 +0,0 @@ -Store the address of the MDB_env structure that owns a reader in addition to -its PID and TID, to allow multiple MDB_envs to be open on the same thread, -since on mdb_env_close(), LMDB unconditionally obliterates any readers with a -matching PID. This patch extends the test to (PID, MDB_env*). - -diff --git a/lib/mdb.c b/lib/mdb.c -index fd0a3b5..f2ebdfa 100644 ---- a/lib/mdb.c -+++ b/lib/mdb.c -@@ -536,6 +536,8 @@ typedef struct MDB_rxbody { - txnid_t mrb_txnid; - /** The process ID of the process owning this reader txn. */ - MDB_PID_T mrb_pid; -+ /** MDB_env within the process owning this reader txn. */ -+ void * mrb_env; - /** The thread ID of the thread owning this txn. */ - pthread_t mrb_tid; - } MDB_rxbody; -@@ -547,6 +549,7 @@ typedef struct MDB_reader { - /** shorthand for mrb_txnid */ - #define mr_txnid mru.mrx.mrb_txnid - #define mr_pid mru.mrx.mrb_pid -+#define mr_env mru.mrx.mrb_env - #define mr_tid mru.mrx.mrb_tid - /** cache line alignment */ - char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; -@@ -2285,6 +2288,7 @@ mdb_txn_renew0(MDB_txn *txn) - return MDB_READERS_FULL; - } - ti->mti_readers[i].mr_pid = pid; -+ ti->mti_readers[i].mr_env = env; - ti->mti_readers[i].mr_tid = tid; - if (i == nr) - ti->mti_numreaders = ++nr; -@@ -4254,7 +4258,8 @@ mdb_env_close0(MDB_env *env, int excl) - * me_txkey with its destructor must be disabled first. - */ - for (i = env->me_numreaders; --i >= 0; ) -- if (env->me_txns->mti_readers[i].mr_pid == pid) -+ if (env->me_txns->mti_readers[i].mr_pid == pid -+ && env->me_txns->mti_readers[i].mr_env == env) - env->me_txns->mti_readers[i].mr_pid = 0; - #ifdef _WIN32 - if (env->me_rmutex) { diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/run_in_vm.py stb-tester-31/vendor/py-lmdb/misc/run_in_vm.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/run_in_vm.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/run_in_vm.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,105 +0,0 @@ -# -# Copyright 2014 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -import atexit -import json -import os -import shutil -import socket -import subprocess -import sys -import tempfile - - -def run(*args): - if os.path.exists('build'): - shutil.rmtree('build') - try: - subprocess.check_call(args) - except: - print '!!! COMMAND WAS:', args - raise - - -def qmp_write(fp, o): - buf = json.dumps(o) + '\n' - fp.write(buf.replace('{', '{ ')) - - -def qmp_read(fp): - s = fp.readline() - return json.loads(s) - - -def qmp_say_hello(fp): - assert 'QMP' in qmp_read(fp) - qmp_write(fp, {'execute': 'qmp_capabilities'}) - assert qmp_read(fp)['return'] == {} - - -def qmp_command(fp, name, args): - qmp_write(fp, {'execute': name, 'arguments': args}) - while True: - o = qmp_read(fp) - if 'return' not in o: - print 'skip', o - continue - print 'cmd out', o - return o['return'] - - -def qmp_monitor(fp, cmd): - return qmp_command(fp, 'human-monitor-command', { - 'command-line': cmd - }) - - -def main(): - vm = sys.argv[1] - cmdline = sys.argv[2:] - - rsock, wsock = socket.socketpair() - rfp = rsock.makefile('r+b', 1) - - qemu_path = '/usr/local/bin/qemu-system-x86_64' - qemu_args = ['sudo', qemu_path, '-enable-kvm', '-m', '1024', - '-qmp', 'stdio', '-nographic', '-S', - '-vnc', '127.0.0.1:0', - '-net', 'user,hostfwd=tcp:127.0.0.1:9422-:22', - '-net', 'nic,model=virtio', - '-drive', 'file=%s,if=virtio' % (vm,)] - print ' '.join(qemu_args).replace('qmp', 'monitor') - exit() - proc = subprocess.Popen(qemu_args, - stdin=wsock.fileno(), stdout=wsock.fileno() - ) - - qmp_say_hello(rfp) - assert '' == qmp_monitor(rfp, 'loadvm 1') - assert '' == qmp_monitor(rfp, 'cont') - import time - time.sleep(100) - qmp_monitor(rfp, 'quit') - -if __name__ == '__main__': - main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/runtests-travisci.sh stb-tester-31/vendor/py-lmdb/misc/runtests-travisci.sh --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/runtests-travisci.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/runtests-travisci.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -#!/bin/bash -ex - -quiet() { - "$@" > /tmp/$$ || { cat /tmp/$$; return 1; } -} - -# Delete Travis PyPy or it'll supercede the PPA version. -rm -rf /usr/local/pypy/bin /usr/local/lib/pypy2.7 -find /usr/lib -name '*setuptools*' | xargs rm -rf -find /usr/local/lib -name '*setuptools*' | xargs rm -rf - -quiet add-apt-repository -y ppa:fkrull/deadsnakes -quiet add-apt-repository -y ppa:pypy -quiet apt-get -qq update -quiet apt-get install --force-yes -qq \ - python{2.5,2.6,2.7,3.1,3.2,3.3}-dev \ - pypy-dev \ - libffi-dev \ - gdb - -wget -qO ez_setup_24.py \ - https://bitbucket.org/pypa/setuptools/raw/bootstrap-py24/ez_setup.py -wget -q https://bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py - -quiet python2.5 ez_setup_24.py -quiet python2.6 ez_setup.py -quiet python2.7 ez_setup.py -quiet python3.1 ez_setup.py -quiet python3.2 ez_setup.py -quiet python3.3 ez_setup.py -quiet pypy ez_setup.py - -quiet python2.5 -measy_install py==1.4.20 pytest==2.5.2 -quiet python2.6 -measy_install py==1.4.20 pytest==2.5.2 cffi -quiet python2.7 -measy_install py==1.4.20 pytest==2.5.2 cffi -quiet python3.1 -measy_install py==1.4.20 pytest==2.5.2 cffi argparse -quiet python3.2 -measy_install py==1.4.20 pytest==2.5.2 cffi -quiet python3.3 -measy_install py==1.4.20 pytest==2.5.2 cffi -quiet pypy -measy_install py==1.4.20 pytest==2.5.2 - -source misc/runtests-ubuntu-12-04.sh diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/runtests-ubuntu-12-04.sh stb-tester-31/vendor/py-lmdb/misc/runtests-ubuntu-12-04.sh --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/runtests-ubuntu-12-04.sh 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/runtests-ubuntu-12-04.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,43 +0,0 @@ -#!/bin/bash -ex - -quiet() { - "$@" > /tmp/$$ || { cat /tmp/$$; return 1; } -} - -clean() { - git clean -qdfx - find /usr/local/lib -name '*lmdb*' | xargs rm -rf - find /usr/lib -name '*lmdb*' | xargs rm -rf -} - -with_gdb() { - gdb --batch -x misc/gdb.commands --args "$@" -} - -native() { - clean - quiet $1 setup.py develop - quiet $1 -c 'import lmdb.cpython' - with_gdb $1 -m pytest tests || fail=1 -} - -cffi() { - clean - LMDB_FORCE_CFFI=1 quiet $1 setup.py install - LMDB_FORCE_CFFI=1 quiet $1 -c 'import lmdb.cffi' - with_gdb $1 -m pytest tests || fail=1 -} - -native python2.5 -native python2.6 -native python2.7 -native python3.3 -cffi pypy -cffi python2.6 -cffi python2.7 -cffi python3.1 -cffi python3.2 -cffi python3.3 - -[ "$fail" ] && exit 1 -exit 0 diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/test_monster_acid_trace.diff stb-tester-31/vendor/py-lmdb/misc/test_monster_acid_trace.diff --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/test_monster_acid_trace.diff 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/test_monster_acid_trace.diff 1970-01-01 00:00:00.000000000 +0000 @@ -1,48 +0,0 @@ -diff --git a/tests/crash_test.py b/tests/crash_test.py -index 1ac3b46..3785bd5 100644 ---- a/tests/crash_test.py -+++ b/tests/crash_test.py -@@ -173,21 +173,43 @@ class MultiCursorDeleteTest(unittest.TestCase): - c2.delete() - assert next(c1f) == B('eeee') - -+ def _trace(self, op, *args): -+ bits = [] -+ for arg in args: -+ if isinstance(arg, bytes): -+ bits.append(arg.encode('hex')) -+ elif isinstance(arg, bool): -+ bits.append(bytes(int(arg))) -+ self.fp.write('%s %s %s\n' % (self.idx, op, ' '.join(bits))) -+ - def test_monster(self): - # Generate predictable sequence of sizes. - rand = random.Random() - rand.seed(0) - -+ self.fp = open('trace.out', 'w') -+ self._counter = 0 -+ self.idx = 0 -+ - txn = self.env.begin(write=True) - keys = [] - for i in range(20000): - key = B('%06x' % i) - val = B('x' * rand.randint(76, 350)) -+ self._trace('put', key, val) - assert txn.put(key, val) - keys.append(key) - -+ -+ iter_id = self._counter -+ self._counter += 1 -+ self._trace('iter', iter_id, '0', False) -+ - deleted = 0 - for key in txn.cursor().iternext(values=False): -+ self._trace('fetch', iter_id) -+ self._trace('yield', iter_id, key, '') -+ self._trace('delete', key) - assert txn.delete(key), key - deleted += 1 - diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/tox.ini stb-tester-31/vendor/py-lmdb/misc/tox.ini --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/tox.ini 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/tox.ini 1970-01-01 00:00:00.000000000 +0000 @@ -1,88 +0,0 @@ -[tox] -; py25_c disabled because TRAVISCI RAGE -envlist = pypy_cffi,py26_cffi,py27_cffi,py30_cffi,py31_cffi,py32_cffi,py33_cffi,py26_c,py27_c,py33_c -skipsdist = True - -[testenv] -install_command = pip install {opts} {packages} -commands = - pip install -e . - python -mpytest ./tests - -[testenv:pypy_cffi] -basepython = pypy -deps = - pytest - -[testenv:py26_cffi] -basepython = python2.6 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py27_cffi] -basepython = python2.7 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py30_cffi] -basepython = python3.0 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py31_cffi] -basepython = python3.1 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py32_cffi] -basepython = python3.2 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py33_cffi] -basepython = python3.3 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py34_cffi] -basepython = python3.4 -deps = - pytest - cffi -setenv = LMDB_FORCE_CFFI=1 - -[testenv:py25_c] -deps = - pytest -setenv = PIP_INSECURE=1 -basepython = python2.5 - -[testenv:py26_c] -deps = - pytest -basepython = python2.6 - -[testenv:py27_c] -deps = - pytest -basepython = python2.7 - -[testenv:py33_c] -deps = - pytest -basepython = python3.3 - -;[testenv:py34_c] -;basepython = python3.4 diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/windows_build.py stb-tester-31/vendor/py-lmdb/misc/windows_build.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/windows_build.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/windows_build.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,92 +0,0 @@ -# -# Copyright 2014 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -import os -import shutil -import subprocess -import tempfile - -INTERPS = ( - ('Python26', False), - ('Python26-64', False), - ('Python27', False), - ('Python27-64', False), - #('Python31', False), - #('Python31-64', False), - ('Python32', False), - ('Python32-64', False), - ('Python33', False), - ('Python33-64', False), - ('Python34', False), - ('Python34-64', False), -) - - -def interp_path(interp): - return r'C:\%s\Python' % (interp,) - -def pip_path(interp): - return os.path.join(os.path.dirname(interp), - 'scripts', 'pip.exe') - -def interp_has_module(path, module): - return run_or_false(path, '-c', 'import ' + module) - - -def run(*args): - if os.path.exists('build'): - shutil.rmtree('build') - try: - subprocess.check_call(args) - except: - print '!!! COMMAND WAS:', args - raise - - -def run_or_false(*args): - try: - run(*args) - except subprocess.CalledProcessError: - return False - return True - - -def main(): - run('git', 'clean', '-dfx', 'dist') - for interp, is_cffi in INTERPS: - path = interp_path(interp) - run('git', 'clean', '-dfx', 'build', 'temp', 'lmdb') - run(pip_path(path), 'install', '-e', '.') - if is_cffi: - os.environ['LMDB_FORCE_CFFI'] = '1' - else: - os.environ.pop('LMDB_FORCE_CFFI', '') - if os.path.exists('lmdb\\cpython.pyd'): - os.unlink('lmdb\\cpython.pyd') - #run(path, '-mpy.test') - run(path, 'setup.py', 'bdist_egg', 'upload') - run(path, 'setup.py', 'bdist_wheel', 'upload') - - -if __name__ == '__main__': - main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/windows_setup.py stb-tester-31/vendor/py-lmdb/misc/windows_setup.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/misc/windows_setup.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/misc/windows_setup.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -# -# Copyright 2014 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -import os -import urllib - -from windows_build import interp_has_module -from windows_build import interp_path -from windows_build import INTERPS -from windows_build import run -from windows_build import run_or_false - -EZSETUP_URL = ('https://bitbucket.org/pypa/setuptools' - '/raw/bootstrap/ez_setup.py') - - -def ezsetup_path(): - path = os.path.join(os.environ['TEMP'], 'ez_setup.py') - if not os.path.exists(path): - fp = urllib.urlopen(EZSETUP_URL) - with open(path, 'wb') as fp2: - fp2.write(fp.read()) - fp.close() - return path - - -def easy_install_path(interp): - return os.path.join(os.path.dirname(interp), - 'scripts', 'easy_install.exe') - - -def main(): - for interp, is_cffi in INTERPS: - path = interp_path(interp) - run_or_false(path, '-m', 'ensurepip') - if not interp_has_module(path, 'easy_install'): - run(path, ezsetup_path()) - for pkg in 'pip', 'cffi', 'pytest', 'wheel': - modname = 'py.test' if pkg == 'pytest' else pkg - if not interp_has_module(path, modname): - run(easy_install_path(path), pkg) - - -if __name__ == '__main__': - main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/README.md stb-tester-31/vendor/py-lmdb/README.md --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/README.md 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,6 +0,0 @@ - -### CI State - -Release: [![release](https://travis-ci.org/dw/py-lmdb.png?branch=release)](https://travis-ci.org/dw/py-lmdb/branches) - -Master: [![master](https://travis-ci.org/dw/py-lmdb.png?branch=master)](https://travis-ci.org/dw/py-lmdb/branches) diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/setup.py stb-tester-31/vendor/py-lmdb/setup.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/setup.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/setup.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,182 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -from __future__ import with_statement - -import os -import sys -import platform - -from setuptools import Extension -from setuptools import setup - -try: - import memsink -except ImportError: - memsink = None - - -if hasattr(platform, 'python_implementation'): - use_cpython = platform.python_implementation() == 'CPython' -else: - use_cpython = True - -if os.getenv('LMDB_FORCE_CFFI') is not None: - use_cpython = False - -if sys.version[:3] < '2.5': - sys.stderr.write('Error: py-lmdb requires at least CPython 2.5\n') - raise SystemExit(1) - -if sys.version[:3] in ('3.0', '3.1', '3.2'): - use_cpython = False - - -# -# Figure out which LMDB implementation to use. -# - -if os.getenv('LMDB_INCLUDEDIR'): - extra_include_dirs = [os.getenv('LMDB_INCLUDEDIR')] -else: - extra_include_dirs = [] - -if os.getenv('LMDB_LIBDIR'): - extra_library_dirs = [os.getenv('LMDB_LIBDIR')] -else: - extra_library_dirs = [] - -extra_include_dirs += ['lib/py-lmdb'] - -if os.getenv('LMDB_FORCE_SYSTEM') is not None: - print('py-lmdb: Using system version of liblmdb.') - extra_sources = [] - extra_include_dirs += [] - libraries = ['lmdb'] -else: - print('py-lmdb: Using bundled liblmdb; override with LMDB_FORCE_SYSTEM=1.') - extra_sources = ['lib/mdb.c', 'lib/midl.c'] - extra_include_dirs += ['lib'] - libraries = [] - - -# distutils perplexingly forces NDEBUG for package code! -extra_compile_args = ['-UNDEBUG'] - -# Disable some Clang/GCC warnings. -if not os.getenv('LMDB_MAINTAINER'): - extra_compile_args += ['-w'] - - -# Microsoft Visual Studio 9 ships with neither inttypes.h, stdint.h, or a sane -# definition for ssize_t, so here we add lib/win32 to the search path, which -# contains emulation header files provided by a third party. We force-include -# Python.h everywhere since it has a portable definition of ssize_t, which -# inttypes.h and stdint.h lack, and to avoid having to modify the LMDB source -# code. Advapi32 is needed for LMDB's use of Windows security APIs. -p = sys.version.find('MSC v.') -msvc_ver = int(sys.version[p+6:p+10]) if p != -1 else None - -if sys.platform.startswith('win'): - # If running on Visual Studio<=2010 we must provide . Newer - # versions provide it out of the box. - if msvc_ver and not msvc_ver >= 1600: - extra_include_dirs += ['lib\\win32-stdint'] - extra_include_dirs += ['lib\\win32'] - extra_compile_args += [r'/FIPython.h'] - libraries += ['Advapi32'] - - -# Capture setup.py configuration for later use by cffi, otherwise the -# configuration may differ, forcing a recompile (and therefore likely compile -# errors). This happens even when `use_cpython` since user might want to -# LMDB_FORCE_CFFI=1 during testing. -with open('lmdb/_config.py', 'w') as fp: - fp.write('CONFIG = %r\n\n' % ({ - 'extra_compile_args': extra_compile_args, - 'extra_sources': extra_sources, - 'extra_library_dirs': extra_library_dirs, - 'extra_include_dirs': extra_include_dirs, - 'libraries': libraries - },)) - - -if use_cpython: - print('py-lmdb: Using CPython extension; override with LMDB_FORCE_CFFI=1.') - install_requires = [] - if memsink: - extra_compile_args += ['-DHAVE_MEMSINK', - '-I' + os.path.dirname(memsink.__file__)] - ext_modules = [Extension( - name='cpython', - sources=['lmdb/cpython.c'] + extra_sources, - extra_compile_args=extra_compile_args, - libraries=libraries, - include_dirs=extra_include_dirs, - library_dirs=extra_library_dirs - )] -else: - print('Using cffi extension.') - install_requires = ['cffi>=0.8'] - try: - import lmdb.cffi - ext_modules = [lmdb.cffi._ffi.verifier.get_extension()] - except ImportError: - sys.stderr.write('Could not import lmdb; ensure cffi is installed!\n') - ext_modules = [] - -def grep_version(): - path = os.path.join(os.path.dirname(__file__), 'lmdb/__init__.py') - with open(path) as fp: - for line in fp: - if line.startswith('__version__'): - return eval(line.split()[-1]) - -setup( - name = 'lmdb', - version = grep_version(), - description = "Universal Python binding for the LMDB 'Lightning' Database", - author = 'David Wilson', - license = 'OpenLDAP BSD', - url = 'http://github.com/dw/py-lmdb/', - packages = ['lmdb'], - classifiers = [ - "Programming Language :: Python", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.0", - "Programming Language :: Python :: 3.1", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", - "Topic :: Database", - "Topic :: Database :: Database Engines/Servers", - ], - ext_package = 'lmdb', - ext_modules = ext_modules, - install_requires = install_requires, - zip_safe = False -) diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/crash_test.py stb-tester-31/vendor/py-lmdb/tests/crash_test.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/crash_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/crash_test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,220 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -# This is not a test suite! More like a collection of triggers for previously -# observed crashes. Want to contribute to py-lmdb? Please write a test suite! -# -# what happens when empty keys/ values passed to various funcs -# incorrect types -# try to break cpython arg parsing - too many/few/incorrect args -# Various efforts to cause Python-level leaks. -# - -from __future__ import absolute_import -from __future__ import with_statement - -import itertools -import os -import random -import unittest - -import lmdb -import testlib - -from testlib import B -from testlib import O - - -try: - next(iter([1])) -except NameError: # Python2.5. - def next(it): - return it.next() - - -class CrashTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - # Various efforts to cause segfaults. - - def setUp(self): - self.path, self.env = testlib.temp_env() - with self.env.begin(write=True) as txn: - txn.put(B('dave'), B('')) - txn.put(B('dave2'), B('')) - - def testOldCrash(self): - txn = self.env.begin() - dir(iter(txn.cursor())) - - def testCloseWithTxn(self): - txn = self.env.begin(write=True) - self.env.close() - self.assertRaises(Exception, (lambda: list(txn.cursor()))) - - def testDoubleClose(self): - self.env.close() - self.env.close() - - def testDbDoubleClose(self): - db = self.env.open_db(key=B('dave3')) - #db.close() - #db.close() - - def testTxnCloseActiveIter(self): - with self.env.begin() as txn: - it = txn.cursor().iternext() - self.assertRaises(Exception, (lambda: list(it))) - - def testDbCloseActiveIter(self): - db = self.env.open_db(key=B('dave3')) - with self.env.begin(write=True) as txn: - txn.put(B('a'), B('b'), db=db) - it = txn.cursor(db=db).iternext() - self.assertRaises(Exception, (lambda: list(it))) - - -class IteratorTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def setUp(self): - self.path, self.env = testlib.temp_env() - self.txn = self.env.begin(write=True) - self.c = self.txn.cursor() - - def testEmpty(self): - self.assertEqual([], list(self.c)) - self.assertEqual([], list(self.c.iternext())) - self.assertEqual([], list(self.c.iterprev())) - - def testFilled(self): - testlib.putData(self.txn) - self.assertEqual(testlib.ITEMS, list(self.c)) - self.assertEqual(testlib.ITEMS, list(self.c)) - self.assertEqual(testlib.ITEMS, list(self.c.iternext())) - self.assertEqual(testlib.ITEMS[::-1], list(self.txn.cursor().iterprev())) - self.assertEqual(testlib.ITEMS[::-1], list(self.c.iterprev())) - self.assertEqual(testlib.ITEMS, list(self.c)) - - def testFilledSkipForward(self): - testlib.putData(self.txn) - self.c.set_range(B('b')) - self.assertEqual(testlib.ITEMS[1:], list(self.c)) - - def testFilledSkipReverse(self): - testlib.putData(self.txn) - self.c.set_range(B('b')) - self.assertEqual(testlib.REV_ITEMS[-2:], list(self.c.iterprev())) - - def testFilledSkipEof(self): - testlib.putData(self.txn) - self.assertEqual(False, self.c.set_range(B('z'))) - self.assertEqual(testlib.REV_ITEMS, list(self.c.iterprev())) - - -class BigReverseTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - # Test for issue with MDB_LAST+MDB_PREV skipping chunks of database. - def test_big_reverse(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - keys = [B('%05d' % i) for i in range(0xffff)] - for k in keys: - txn.put(k, k, append=True) - assert list(txn.cursor().iterprev(values=False)) == list(reversed(keys)) - - -class MultiCursorDeleteTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def setUp(self): - self.path, self.env = testlib.temp_env() - - def test1(self): - """Ensure MDB_NEXT is ignored on `c1' when it was previously positioned - on the key that `c2' just deleted.""" - txn = self.env.begin(write=True) - cur = txn.cursor() - while cur.first(): - cur.delete() - - for i in range(1, 10): - cur.put(O(ord('a') + i) * i, B('')) - - c1 = txn.cursor() - c1f = c1.iternext(values=False) - while next(c1f) != B('ddd'): - pass - c2 = txn.cursor() - assert c2.set_key(B('ddd')) - c2.delete() - assert next(c1f) == B('eeee') - - def test_monster(self): - # Generate predictable sequence of sizes. - rand = random.Random() - rand.seed(0) - - txn = self.env.begin(write=True) - keys = [] - for i in range(20000): - key = B('%06x' % i) - val = B('x' * rand.randint(76, 350)) - assert txn.put(key, val) - keys.append(key) - - deleted = 0 - for key in txn.cursor().iternext(values=False): - assert txn.delete(key), key - deleted += 1 - - assert deleted == len(keys), deleted - - -class TxnFullTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_17bf75b12eb94d9903cd62329048b146d5313bad(self): - """ - me_txn0 previously cached MDB_TXN_ERROR permanently. Fixed by - 17bf75b12eb94d9903cd62329048b146d5313bad. - """ - path, env = testlib.temp_env(map_size=4096*9, sync=False, max_spare_txns=0) - for i in itertools.count(): - try: - with env.begin(write=True) as txn: - txn.put(B(str(i)), B(str(i))) - except lmdb.MapFullError: - break - - # Should not crash with MDB_BAD_TXN: - with env.begin(write=True) as txn: - txn.delete(B('1')) - -if __name__ == '__main__': - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/cursor_test.py stb-tester-31/vendor/py-lmdb/tests/cursor_test.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/cursor_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/cursor_test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,222 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -# test delete(dupdata) - -from __future__ import absolute_import -from __future__ import with_statement -import unittest - -import testlib -from testlib import B -from testlib import BT - - -class ContextManagerTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_ok(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - with txn.cursor() as curs: - curs.put(B('foo'), B('123')) - self.assertRaises(Exception, lambda: curs.get(B('foo'))) - - def test_crash(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - - try: - with txn.cursor() as curs: - curs.put(123, 123) - except: - pass - self.assertRaises(Exception, lambda: curs.get(B('foo'))) - - -class CursorTestBase(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def setUp(self): - self.path, self.env = testlib.temp_env() - self.txn = self.env.begin(write=True) - self.c = self.txn.cursor() - - -class CursorTest(CursorTestBase): - def testKeyValueItemEmpty(self): - self.assertEqual(B(''), self.c.key()) - self.assertEqual(B(''), self.c.value()) - self.assertEqual(BT('', ''), self.c.item()) - - def testFirstLastEmpty(self): - self.assertEqual(False, self.c.first()) - self.assertEqual(False, self.c.last()) - - def testFirstFilled(self): - testlib.putData(self.txn) - self.assertEqual(True, self.c.first()) - self.assertEqual(testlib.ITEMS[0], self.c.item()) - - def testLastFilled(self): - testlib.putData(self.txn) - self.assertEqual(True, self.c.last()) - self.assertEqual(testlib.ITEMS[-1], self.c.item()) - - def testSetKey(self): - self.assertRaises(Exception, (lambda: self.c.set_key(B('')))) - self.assertEqual(False, self.c.set_key(B('missing'))) - testlib.putData(self.txn) - self.assertEqual(True, self.c.set_key(B('b'))) - self.assertEqual(False, self.c.set_key(B('ba'))) - - def testSetRange(self): - self.assertEqual(False, self.c.set_range(B('x'))) - testlib.putData(self.txn) - self.assertEqual(False, self.c.set_range(B('x'))) - self.assertEqual(True, self.c.set_range(B('a'))) - self.assertEqual(B('a'), self.c.key()) - self.assertEqual(True, self.c.set_range(B('ba'))) - self.assertEqual(B('baa'), self.c.key()) - self.c.set_range(B('')) - self.assertEqual(B('a'), self.c.key()) - - def testDeleteEmpty(self): - self.assertEqual(False, self.c.delete()) - - def testDeleteFirst(self): - testlib.putData(self.txn) - self.assertEqual(False, self.c.delete()) - self.c.first() - self.assertEqual(BT('a', ''), self.c.item()) - self.assertEqual(True, self.c.delete()) - self.assertEqual(BT('b', ''), self.c.item()) - self.assertEqual(True, self.c.delete()) - self.assertEqual(BT('baa', ''), self.c.item()) - self.assertEqual(True, self.c.delete()) - self.assertEqual(BT('d', ''), self.c.item()) - self.assertEqual(True, self.c.delete()) - self.assertEqual(BT('', ''), self.c.item()) - self.assertEqual(False, self.c.delete()) - self.assertEqual(BT('', ''), self.c.item()) - - def testDeleteLast(self): - testlib.putData(self.txn) - self.assertEqual(True, self.c.last()) - self.assertEqual(BT('d', ''), self.c.item()) - self.assertEqual(True, self.c.delete()) - self.assertEqual(BT('', ''), self.c.item()) - self.assertEqual(False, self.c.delete()) - self.assertEqual(BT('', ''), self.c.item()) - - def testCount(self): - self.assertRaises(Exception, (lambda: self.c.count())) - testlib.putData(self.txn) - self.c.first() - # TODO: complete dup key support. - #self.assertEqual(1, self.c.count()) - - def testPut(self): - pass - - -class PutmultiTest(CursorTestBase): - def test_empty_seq(self): - consumed, added = self.c.putmulti(()) - assert consumed == added == 0 - - def test_2list(self): - l = [BT('a', ''), BT('a', '')] - consumed, added = self.c.putmulti(l) - assert consumed == added == 2 - - li = iter(l) - consumed, added = self.c.putmulti(li) - assert consumed == added == 2 - - def test_2list_preserve(self): - l = [BT('a', ''), BT('a', '')] - consumed, added = self.c.putmulti(l, overwrite=False) - assert consumed == 2 - assert added == 1 - - assert self.c.set_key(B('a')) - assert self.c.delete() - - li = iter(l) - consumed, added = self.c.putmulti(li, overwrite=False) - assert consumed == 2 - assert added == 1 - - def test_bad_seq1(self): - self.assertRaises(Exception, - lambda: self.c.putmulti(range(2))) - - -class ReplaceTest(CursorTestBase): - def test_replace(self): - assert None is self.c.replace(B('a'), B('')) - assert B('') == self.c.replace(B('a'), B('x')) - assert B('x') == self.c.replace(B('a'), B('y')) - - -class ContextManagerTest(CursorTestBase): - def test_enter(self): - with self.c as c: - assert c is self.c - c.put(B('a'), B('a')) - assert c.get(B('a')) == B('a') - self.assertRaises(Exception, - lambda: c.get(B('a'))) - - def test_exit_success(self): - with self.txn.cursor() as c: - c.put(B('a'), B('a')) - self.assertRaises(Exception, - lambda: c.get(B('a'))) - - def test_exit_failure(self): - try: - with self.txn.cursor() as c: - c.put(B('a'), B('a')) - raise ValueError - except ValueError: - pass - self.assertRaises(Exception, - lambda: c.get(B('a'))) - - def test_close(self): - self.c.close() - self.assertRaises(Exception, - lambda: c.get(B('a'))) - - def test_double_close(self): - self.c.close() - self.c.close() - self.assertRaises(Exception, - lambda: self.c.put(B('a'), B('a'))) - - -if __name__ == '__main__': - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/env_test.py stb-tester-31/vendor/py-lmdb/tests/env_test.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/env_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/env_test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,753 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -from __future__ import with_statement -import os -import signal -import sys -import unittest -import weakref - -import testlib -from testlib import B -from testlib import BT -from testlib import OCT -from testlib import INT_TYPES -from testlib import UnicodeType - -import lmdb - - -NO_READERS = UnicodeType('(no active readers)\n') - - -class VersionTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_version(self): - ver = lmdb.version() - assert len(ver) == 3 - assert all(isinstance(i, INT_TYPES) for i in ver) - assert all(i >= 0 for i in ver) - - -class OpenTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_bad_paths(self): - self.assertRaises(Exception, - lambda: lmdb.open('/doesnt/exist/at/all')) - self.assertRaises(Exception, - lambda: lmdb.open(testlib.temp_file())) - - def test_ok_path(self): - path, env = testlib.temp_env() - assert os.path.exists(path) - assert os.path.exists(os.path.join(path, 'data.mdb')) - assert os.path.exists(os.path.join(path, 'lock.mdb')) - assert env.path() == path - - def test_bad_size(self): - self.assertRaises(OverflowError, - lambda: testlib.temp_env(map_size=-123)) - - def test_subdir_false_junk(self): - path = testlib.temp_file() - fp = open(path, 'wb') - fp.write(B('A' * 8192)) - fp.close() - self.assertRaises(lmdb.InvalidError, - lambda: lmdb.open(path, subdir=False)) - - def test_subdir_false_ok(self): - path = testlib.temp_file(create=False) - _, env = testlib.temp_env(path, subdir=False) - assert os.path.exists(path) - assert os.path.isfile(path) - assert os.path.isfile(path + '-lock') - assert not env.flags()['subdir'] - - def test_subdir_true_noexist_nocreate(self): - path = testlib.temp_dir(create=False) - self.assertRaises(lmdb.Error, - lambda: testlib.temp_env(path, subdir=True, create=False)) - assert not os.path.exists(path) - - def test_subdir_true_noexist_create(self): - path = testlib.temp_dir(create=False) - path_, env = testlib.temp_env(path, subdir=True, create=True) - assert path_ == path - assert env.path() == path - - def test_subdir_true_exist_nocreate(self): - path, env = testlib.temp_env() - assert lmdb.open(path, subdir=True, create=False).path() == path - - def test_subdir_true_exist_create(self): - path, env = testlib.temp_env() - assert lmdb.open(path, subdir=True, create=True).path() == path - - def test_readonly_false(self): - path, env = testlib.temp_env(readonly=False) - with env.begin(write=True) as txn: - txn.put(B('a'), B('')) - with env.begin() as txn: - assert txn.get(B('a')) == B('') - assert not env.flags()['readonly'] - - def test_readonly_true_noexist(self): - path = testlib.temp_dir(create=False) - # Open readonly missing store should fail. - self.assertRaises(lmdb.Error, - lambda: lmdb.open(path, readonly=True, create=True)) - # And create=True should not have mkdir'd it. - assert not os.path.exists(path) - - def test_readonly_true_exist(self): - path, env = testlib.temp_env() - env2 = lmdb.open(path, readonly=True) - assert env2.path() == path - # Attempting a write txn should fail. - self.assertRaises(lmdb.ReadonlyError, - lambda: env2.begin(write=True)) - # Flag should be set. - assert env2.flags()['readonly'] - - def test_metasync(self): - for flag in True, False: - path, env = testlib.temp_env(metasync=flag) - assert env.flags()['metasync'] == flag - - def test_lock(self): - for flag in True, False: - path, env = testlib.temp_env(lock=flag) - lock_path = os.path.join(path, 'lock.mdb') - assert env.flags()['lock'] == flag - assert flag == os.path.exists(lock_path) - - def test_sync(self): - for flag in True, False: - path, env = testlib.temp_env(sync=flag) - assert env.flags()['sync'] == flag - - def test_map_async(self): - for flag in True, False: - path, env = testlib.temp_env(map_async=flag) - assert env.flags()['map_async'] == flag - - def test_mode_subdir_create(self): - oldmask = os.umask(0) - try: - for mode in OCT('777'), OCT('755'), OCT('700'): - path = testlib.temp_dir(create=False) - env = lmdb.open(path, subdir=True, create=True, mode=mode) - fmode = mode & ~OCT('111') - assert testlib.path_mode(path) == mode - assert testlib.path_mode(path+'/data.mdb') == fmode - assert testlib.path_mode(path+'/lock.mdb') == fmode - finally: - os.umask(oldmask) - - def test_mode_subdir_nocreate(self): - oldmask = os.umask(0) - try: - for mode in OCT('777'), OCT('755'), OCT('700'): - path = testlib.temp_dir() - env = lmdb.open(path, subdir=True, create=False, mode=mode) - fmode = mode & ~OCT('111') - assert testlib.path_mode(path+'/data.mdb') == fmode - assert testlib.path_mode(path+'/lock.mdb') == fmode - finally: - os.umask(oldmask) - - def test_readahead(self): - for flag in True, False: - path, env = testlib.temp_env(readahead=flag) - assert env.flags()['readahead'] == flag - - def test_writemap(self): - for flag in True, False: - path, env = testlib.temp_env(writemap=flag) - assert env.flags()['writemap'] == flag - - def test_meminit(self): - for flag in True, False: - path, env = testlib.temp_env(meminit=flag) - assert env.flags()['meminit'] == flag - - def test_max_readers(self): - self.assertRaises(lmdb.InvalidParameterError, - lambda: testlib.temp_env(max_readers=0)) - for val in 123, 234: - _, env = testlib.temp_env(max_readers=val) - assert env.info()['max_readers'] == val - - def test_max_dbs(self): - self.assertRaises(OverflowError, - lambda: testlib.temp_env(max_dbs=-1)) - for val in 0, 10, 20: - _, env = testlib.temp_env(max_dbs=val) - dbs = [env.open_db(B('db%d' % i)) for i in range(val)] - self.assertRaises(lmdb.DbsFullError, - lambda: env.open_db(B('toomany'))) - - -class CloseTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_close(self): - _, env = testlib.temp_env() - # Attempting things should be ok. - txn = env.begin(write=True) - txn.put(B('a'), B('')) - cursor = txn.cursor() - list(cursor) - cursor.first() - it = iter(cursor) - - env.close() - # Repeated calls are ignored: - env.close() - # Attempting to use invalid objects should crash. - self.assertRaises(Exception, lambda: txn.cursor()) - self.assertRaises(Exception, lambda: txn.commit()) - self.assertRaises(Exception, lambda: cursor.first()) - self.assertRaises(Exception, lambda: list(it)) - # Abort should be OK though. - txn.abort() - # Attempting to start new txn should crash. - self.assertRaises(Exception, - lambda: env.begin()) - - -class ContextManagerTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_ok(self): - path, env = testlib.temp_env() - with env as env_: - assert env_ is env - with env.begin() as txn: - txn.get(B('foo')) - self.assertRaises(Exception, lambda: env.begin()) - - def test_crash(self): - path, env = testlib.temp_env() - try: - with env as env_: - assert env_ is env - with env.begin() as txn: - txn.get(123) - except: - pass - self.assertRaises(Exception, lambda: env.begin()) - - -class InfoMethodsTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_path(self): - path, env = testlib.temp_env() - assert path == env.path() - assert isinstance(env.path(), UnicodeType) - - env.close() - self.assertRaises(Exception, - lambda: env.path()) - - def test_stat(self): - _, env = testlib.temp_env() - stat = env.stat() - for k in 'psize', 'depth', 'branch_pages', 'overflow_pages',\ - 'entries': - assert isinstance(stat[k], INT_TYPES), k - assert stat[k] >= 0 - - assert stat['entries'] == 0 - txn = env.begin(write=True) - txn.put(B('a'), B('b')) - txn.commit() - stat = env.stat() - assert stat['entries'] == 1 - - env.close() - self.assertRaises(Exception, - lambda: env.stat()) - - def test_info(self): - _, env = testlib.temp_env() - info = env.info() - for k in 'map_addr', 'map_size', 'last_pgno', 'last_txnid', \ - 'max_readers', 'num_readers': - assert isinstance(info[k], INT_TYPES), k - assert info[k] >= 0 - - assert info['last_txnid'] == 0 - txn = env.begin(write=True) - txn.put(B('a'), B('')) - txn.commit() - info = env.info() - assert info['last_txnid'] == 1 - - env.close() - self.assertRaises(Exception, - lambda: env.info()) - - def test_flags(self): - _, env = testlib.temp_env() - info = env.flags() - for k in 'subdir', 'readonly', 'metasync', 'sync', 'map_async',\ - 'readahead', 'writemap': - assert isinstance(info[k], bool) - - env.close() - self.assertRaises(Exception, - lambda: env.flags()) - - def test_max_key_size(self): - _, env = testlib.temp_env() - mks = env.max_key_size() - assert isinstance(mks, INT_TYPES) - assert mks > 0 - - env.close() - self.assertRaises(Exception, - lambda: env.max_key_size()) - - def test_max_readers(self): - _, env = testlib.temp_env() - mr = env.max_readers() - assert isinstance(mr, INT_TYPES) - assert mr > 0 and mr == env.info()['max_readers'] - - env.close() - self.assertRaises(Exception, - lambda: env.max_readers()) - - def test_readers(self): - _, env = testlib.temp_env(max_spare_txns=0) - r = env.readers() - assert isinstance(r, UnicodeType) - assert r == NO_READERS - - rtxn = env.begin() - r2 = env.readers() - assert isinstance(env.readers(), UnicodeType) - assert env.readers() != r - - env.close() - self.assertRaises(Exception, - lambda: env.readers()) - - -class OtherMethodsTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_copy(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - txn.put(B('a'), B('b')) - txn.commit() - - dest_dir = testlib.temp_dir() - env.copy(dest_dir) - assert os.path.exists(dest_dir + '/data.mdb') - - cenv = lmdb.open(dest_dir) - ctxn = cenv.begin() - assert ctxn.get(B('a')) == B('b') - - env.close() - self.assertRaises(Exception, - lambda: env.copy(testlib.temp_dir())) - - def test_copy_compact(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - txn.put(B('a'), B('b')) - txn.commit() - - dest_dir = testlib.temp_dir() - env.copy(dest_dir, compact=True) - assert os.path.exists(dest_dir + '/data.mdb') - - cenv = lmdb.open(dest_dir) - ctxn = cenv.begin() - assert ctxn.get(B('a')) == B('b') - - env.close() - self.assertRaises(Exception, - lambda: env.copy(testlib.temp_dir())) - - def test_copyfd(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - txn.put(B('a'), B('b')) - txn.commit() - - dst_path = testlib.temp_file(create=False) - fp = open(dst_path, 'wb') - env.copyfd(fp.fileno()) - - dstenv = lmdb.open(dst_path, subdir=False) - dtxn = dstenv.begin() - assert dtxn.get(B('a')) == B('b') - - env.close() - self.assertRaises(Exception, - lambda: env.copyfd(fp.fileno())) - fp.close() - - def test_copyfd_compact(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - txn.put(B('a'), B('b')) - txn.commit() - - dst_path = testlib.temp_file(create=False) - fp = open(dst_path, 'wb') - env.copyfd(fp.fileno(), compact=True) - - dstenv = lmdb.open(dst_path, subdir=False) - dtxn = dstenv.begin() - assert dtxn.get(B('a')) == B('b') - - env.close() - self.assertRaises(Exception, - lambda: env.copyfd(fp.fileno())) - fp.close() - - def test_sync(self): - _, env = testlib.temp_env() - env.sync(False) - env.sync(True) - env.close() - self.assertRaises(Exception, - lambda: env.sync(False)) - - @staticmethod - def _test_reader_check_child(path): - """Function to run in child process since we can't use fork() on - win32.""" - env = lmdb.open(path, max_spare_txns=0) - txn = env.begin() - os._exit(0) - - def test_reader_check(self): - path, env = testlib.temp_env(max_spare_txns=0) - rc = env.reader_check() - assert rc == 0 - - # We need to open a separate env since Transaction.abort() always calls - # reset for a read-only txn, the actual abort doesn't happen until - # __del__, when Transaction discovers there is no room for it on the - # freelist. - env1 = lmdb.open(path) - txn1 = env1.begin() - assert env.readers() != NO_READERS - assert env.reader_check() == 0 - - # Start a child, open a txn, then crash the child. - rc = os.spawnl(os.P_WAIT, sys.executable, sys.executable, - __file__, 'test_reader_check_child', path) - - assert rc == 0 - assert env.reader_check() == 1 - assert env.reader_check() == 0 - assert env.readers() != NO_READERS - - txn1.abort() - env1.close() - assert env.readers() == NO_READERS - - env.close() - self.assertRaises(Exception, - lambda: env.reader_check()) - - -class BeginTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_begin_closed(self): - _, env = testlib.temp_env() - env.close() - self.assertRaises(Exception, - lambda: env.begin()) - - def test_begin_readonly(self): - _, env = testlib.temp_env() - txn = env.begin() - # Read txn can't write. - self.assertRaises(lmdb.ReadonlyError, - lambda: txn.put(B('a'), B(''))) - txn.abort() - - def test_begin_write(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - # Write txn can write. - assert txn.put(B('a'), B('')) - txn.commit() - - def test_bind_db(self): - _, env = testlib.temp_env() - main = env.open_db(None) - sub = env.open_db(B('db1')) - - txn = env.begin(write=True, db=sub) - assert txn.put(B('b'), B('')) # -> sub - assert txn.put(B('a'), B(''), db=main) # -> main - txn.commit() - - txn = env.begin() - assert txn.get(B('a')) == B('') - assert txn.get(B('b')) is None - assert txn.get(B('a'), db=sub) is None - assert txn.get(B('b'), db=sub) == B('') - txn.abort() - - def test_parent_readonly(self): - _, env = testlib.temp_env() - parent = env.begin() - # Nonsensical. - self.assertRaises(lmdb.InvalidParameterError, - lambda: env.begin(parent=parent)) - - def test_parent(self): - _, env = testlib.temp_env() - parent = env.begin(write=True) - parent.put(B('a'), B('a')) - - child = env.begin(write=True, parent=parent) - assert child.get(B('a')) == B('a') - assert child.put(B('a'), B('b')) - child.abort() - - # put() should have rolled back - assert parent.get(B('a')) == B('a') - - child = env.begin(write=True, parent=parent) - assert child.put(B('a'), B('b')) - child.commit() - - # put() should be visible - assert parent.get(B('a')) == B('b') - - def test_buffers(self): - _, env = testlib.temp_env() - txn = env.begin(write=True, buffers=True) - assert txn.put(B('a'), B('a')) - b = txn.get(B('a')) - assert b is not None - assert len(b) == 1 - assert not isinstance(b, type(B(''))) - txn.commit() - - txn = env.begin(buffers=False) - b = txn.get(B('a')) - assert b is not None - assert len(b) == 1 - assert isinstance(b, type(B(''))) - txn.abort() - - -class OpenDbTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_main(self): - _, env = testlib.temp_env() - # Start write txn, so we cause deadlock if open_db attempts txn. - txn = env.begin(write=True) - # Now get main DBI, we should already be open. - db = env.open_db(None) - # w00t, no deadlock. - - flags = db.flags(txn) - assert not flags['reverse_key'] - assert not flags['dupsort'] - txn.abort() - - def test_unicode(self): - _, env = testlib.temp_env() - assert env.open_db(B('myindex')) is not None - self.assertRaises(TypeError, - lambda: env.open_db(UnicodeType('myindex'))) - - def test_sub_notxn(self): - _, env = testlib.temp_env() - assert env.info()['last_txnid'] == 0 - db1 = env.open_db(B('subdb1')) - assert env.info()['last_txnid'] == 1 - db2 = env.open_db(B('subdb2')) - assert env.info()['last_txnid'] == 2 - - env.close() - self.assertRaises(Exception, - lambda: env.open_db('subdb3')) - - def test_sub_rotxn(self): - _, env = testlib.temp_env() - txn = env.begin(write=False) - self.assertRaises(lmdb.ReadonlyError, - lambda: env.open_db(B('subdb'), txn=txn)) - - def test_sub_txn(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - db1 = env.open_db(B('subdb1'), txn=txn) - db2 = env.open_db(B('subdb2'), txn=txn) - for db in db1, db2: - assert db.flags(txn) == {'reverse_key': False, 'dupsort': False} - txn.commit() - - def test_reopen(self): - path, env = testlib.temp_env() - db1 = env.open_db(B('subdb1')) - env.close() - env = lmdb.open(path, max_dbs=10) - db1 = env.open_db(B('subdb1')) - - FLAG_SETS = [(flag, val) - for flag in ('reverse_key', 'dupsort') - for val in (True, False)] - - def test_flags(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - - for flag, val in self.FLAG_SETS: - key = B('%s-%s' % (flag, val)) - db = env.open_db(key, txn=txn, **{flag: val}) - assert db.flags(txn)[flag] == val - - txn.commit() - # Test flag persistence. - env.close() - env = lmdb.open(path, max_dbs=10) - txn = env.begin(write=True) - - for flag, val in self.FLAG_SETS: - key = B('%s-%s' % (flag, val)) - db = env.open_db(key, txn=txn) - assert db.flags(txn)[flag] == val - - -reader_count = lambda env: env.readers().count('\n') - 1 - -class SpareTxnTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_none(self): - _, env = testlib.temp_env(max_spare_txns=0) - assert 0 == reader_count(env) - - t1 = env.begin() - assert 1 == reader_count(env) - - t2 = env.begin() - assert 2 == reader_count(env) - - t1.abort() - del t1 - assert 1 == reader_count(env) - - t2.abort() - del t2 - assert 0 == reader_count(env) - - def test_one(self): - _, env = testlib.temp_env(max_spare_txns=1) - # 1 here, since CFFI uses a temporary reader during init. - assert 1 >= reader_count(env) - - t1 = env.begin() - assert 1 == reader_count(env) - - t2 = env.begin() - assert 2 == reader_count(env) - - t1.abort() - del t1 - assert 2 == reader_count(env) # 1 live, 1 cached - - t2.abort() - del t2 - assert 1 == reader_count(env) # 1 cached - - t3 = env.begin() - assert 1 == reader_count(env) # 1 live - - t3.abort() - del t3 - assert 1 == reader_count(env) # 1 cached - - -class LeakTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_open_unref_does_not_leak(self): - temp_dir = testlib.temp_dir() - env = lmdb.open(temp_dir) - ref = weakref.ref(env) - env = None - testlib.debug_collect() - assert ref() is None - - def test_open_close_does_not_leak(self): - temp_dir = testlib.temp_dir() - env = lmdb.open(temp_dir) - env.close() - ref = weakref.ref(env) - env = None - testlib.debug_collect() - assert ref() is None - - def test_weakref_callback_invoked_once(self): - temp_dir = testlib.temp_dir() - env = lmdb.open(temp_dir) - env.close() - count = [0] - def callback(ref): - count[0] += 1 - ref = weakref.ref(env, callback) - env = None - testlib.debug_collect() - assert ref() is None - assert count[0] == 1 - - -if __name__ == '__main__': - if len(sys.argv) > 1 and sys.argv[1] == 'test_reader_check_child': - OtherMethodsTest._test_reader_check_child(sys.argv[2]) - else: - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/testlib.py stb-tester-31/vendor/py-lmdb/tests/testlib.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/testlib.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/testlib.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,142 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -import atexit -import gc -import os -import shutil -import stat -import sys -import tempfile -import traceback - -try: - import __builtin__ -except ImportError: - import builtins as __builtin__ - -import lmdb - - -_cleanups = [] - -def cleanup(): - while _cleanups: - func = _cleanups.pop() - try: - func() - except Exception: - traceback.print_exc() - -atexit.register(cleanup) - - -def temp_dir(create=True): - path = tempfile.mkdtemp(prefix='lmdb_test') - assert path is not None, 'tempfile.mkdtemp failed' - if not create: - os.rmdir(path) - _cleanups.append(lambda: shutil.rmtree(path, ignore_errors=True)) - if hasattr(path, 'decode'): - path = path.decode(sys.getfilesystemencoding()) - return path - - -def temp_file(create=True): - fd, path = tempfile.mkstemp(prefix='lmdb_test') - assert path is not None, 'tempfile.mkstemp failed' - os.close(fd) - if not create: - os.unlink(path) - _cleanups.append(lambda: os.path.exists(path) and os.unlink(path)) - pathlock = path + '-lock' - _cleanups.append(lambda: os.path.exists(pathlock) and os.unlink(pathlock)) - if hasattr(path, 'decode'): - path = path.decode(sys.getfilesystemencoding()) - return path - - -def temp_env(path=None, max_dbs=10, **kwargs): - if not path: - path = temp_dir() - env = lmdb.open(path, max_dbs=max_dbs, **kwargs) - _cleanups.append(env.close) - return path, env - - -def path_mode(path): - return stat.S_IMODE(os.stat(path).st_mode) - - -def debug_collect(): - if hasattr(gc, 'set_debug') and hasattr(gc, 'get_debug'): - old = gc.get_debug() - gc.set_debug(gc.DEBUG_LEAK) - gc.collect() - gc.set_debug(old) - else: - for x in range(10): - # PyPy doesn't collect objects with __del__ on first attempt. - gc.collect() - - -# Handle moronic Python >=3.0 <3.3. -UnicodeType = getattr(__builtin__, 'unicode', str) -BytesType = getattr(__builtin__, 'bytes', str) - - -try: - INT_TYPES = (int, long) -except NameError: - INT_TYPES = (int,) - -# B(ascii 'string') -> bytes -try: - bytes('') # Python>=2.6, alias for str(). - B = lambda s: s -except TypeError: # Python3.x, requires encoding parameter. - B = lambda s: bytes(s, 'ascii') -except NameError: # Python<=2.5. - B = lambda s: s - -# BL('s1', 's2') -> ['bytes1', 'bytes2'] -BL = lambda *args: map(B, args) -# TS('s1', 's2') -> ('bytes1', 'bytes2') -BT = lambda *args: tuple(B(s) for s in args) -# O(int) -> length-1 bytes -O = lambda arg: B(chr(arg)) -# OCT(s) -> parse string as octal -OCT = lambda s: int(s, 8) - - -KEYS = BL('a', 'b', 'baa', 'd') -ITEMS = [(k, B('')) for k in KEYS] -REV_ITEMS = ITEMS[::-1] -VALUES = [B('') for k in KEYS] - -def putData(t, db=None): - for k, v in ITEMS: - if db: - t.put(k, v, db=db) - else: - t.put(k, v) diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/tool_test.py stb-tester-31/vendor/py-lmdb/tests/tool_test.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/tool_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/tool_test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -import unittest - -import lmdb -import lmdb.tool - - -class ToolTest(unittest.TestCase): - def test_ok(self): - # For now, simply ensure the module can be compiled (3.x compat). - pass - - -if __name__ == '__main__': - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/txn_test.py stb-tester-31/vendor/py-lmdb/tests/txn_test.py --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/tests/txn_test.py 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/tests/txn_test.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,478 +0,0 @@ -# -# Copyright 2013 The py-lmdb authors, all rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted only as authorized by the OpenLDAP -# Public License. -# -# A copy of this license is available in the file LICENSE in the -# top-level directory of the distribution or, alternatively, at -# . -# -# OpenLDAP is a registered trademark of the OpenLDAP Foundation. -# -# Individual files and/or contributed packages may be copyright by -# other parties and/or subject to additional restrictions. -# -# This work also contains materials derived from public sources. -# -# Additional information about OpenLDAP can be obtained at -# . -# - -from __future__ import absolute_import -from __future__ import with_statement -import unittest -import weakref - -import testlib -from testlib import B -from testlib import BT -from testlib import OCT -from testlib import INT_TYPES -from testlib import BytesType -from testlib import UnicodeType - -import lmdb - - -class InitTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_closed(self): - _, env = testlib.temp_env() - env.close() - self.assertRaises(Exception, - lambda: lmdb.Transaction(env)) - - def test_readonly(self): - _, env = testlib.temp_env() - txn = lmdb.Transaction(env) - # Read txn can't write. - self.assertRaises(lmdb.ReadonlyError, - lambda: txn.put(B('a'), B(''))) - txn.abort() - - def test_begin_write(self): - _, env = testlib.temp_env() - txn = lmdb.Transaction(env, write=True) - # Write txn can write. - assert txn.put(B('a'), B('')) - txn.commit() - - def test_bind_db(self): - _, env = testlib.temp_env() - main = env.open_db(None) - sub = env.open_db(B('db1')) - - txn = lmdb.Transaction(env, write=True, db=sub) - assert txn.put(B('b'), B('')) # -> sub - assert txn.put(B('a'), B(''), db=main) # -> main - txn.commit() - - txn = lmdb.Transaction(env) - assert txn.get(B('a')) == B('') - assert txn.get(B('b')) is None - assert txn.get(B('a'), db=sub) is None - assert txn.get(B('b'), db=sub) == B('') - txn.abort() - - def test_bind_db_methods(self): - _, env = testlib.temp_env() - maindb = env.open_db(None) - db1 = env.open_db(B('d1')) - txn = lmdb.Transaction(env, write=True, db=db1) - assert txn.put(B('a'), B('d1')) - assert txn.get(B('a'), db=db1) == B('d1') - assert txn.get(B('a'), db=maindb) is None - assert txn.replace(B('a'), B('d11')) == B('d1') - assert txn.pop(B('a')) == B('d11') - assert txn.put(B('a'), B('main'), db=maindb, overwrite=False) - assert not txn.delete(B('a')) - txn.abort() - - def test_parent_readonly(self): - _, env = testlib.temp_env() - parent = lmdb.Transaction(env) - # Nonsensical. - self.assertRaises(lmdb.InvalidParameterError, - lambda: lmdb.Transaction(env, parent=parent)) - - def test_parent(self): - _, env = testlib.temp_env() - parent = lmdb.Transaction(env, write=True) - parent.put(B('a'), B('a')) - - child = lmdb.Transaction(env, write=True, parent=parent) - assert child.get(B('a')) == B('a') - assert child.put(B('a'), B('b')) - child.abort() - - # put() should have rolled back - assert parent.get(B('a')) == B('a') - - child = lmdb.Transaction(env, write=True, parent=parent) - assert child.put(B('a'), B('b')) - child.commit() - - # put() should be visible - assert parent.get(B('a')) == B('b') - - def test_buffers(self): - _, env = testlib.temp_env() - txn = lmdb.Transaction(env, write=True, buffers=True) - assert txn.put(B('a'), B('a')) - b = txn.get(B('a')) - assert b is not None - assert len(b) == 1 - assert not isinstance(b, type(B(''))) - txn.commit() - - txn = lmdb.Transaction(env, buffers=False) - b = txn.get(B('a')) - assert b is not None - assert len(b) == 1 - assert isinstance(b, type(B(''))) - txn.abort() - - -class ContextManagerTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_ok(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - with txn as txn_: - assert txn is txn_ - txn.put(B('foo'), B('123')) - - self.assertRaises(Exception, lambda: txn.get(B('foo'))) - with env.begin() as txn: - assert txn.get(B('foo')) == B('123') - - def test_crash(self): - path, env = testlib.temp_env() - txn = env.begin(write=True) - - try: - with txn as txn_: - txn.put(B('foo'), B('123')) - txn.put(123, 123) - except: - pass - - self.assertRaises(Exception, lambda: txn.get(B('foo'))) - with env.begin() as txn: - assert txn.get(B('foo')) is None - - -class StatTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_stat(self): - _, env = testlib.temp_env() - db1 = env.open_db(B('db1')) - db2 = env.open_db(B('db2')) - - txn = lmdb.Transaction(env) - for db in db1, db2: - stat = txn.stat(db) - for k in 'psize', 'depth', 'branch_pages', 'overflow_pages',\ - 'entries': - assert isinstance(stat[k], INT_TYPES), k - assert stat[k] >= 0 - assert stat['entries'] == 0 - - txn = lmdb.Transaction(env, write=True) - txn.put(B('a'), B('b'), db=db1) - txn.commit() - - txn = lmdb.Transaction(env) - stat = txn.stat(db1) - assert stat['entries'] == 1 - - stat = txn.stat(db2) - assert stat['entries'] == 0 - - txn.abort() - self.assertRaises(Exception, - lambda: env.stat(db1)) - env.close() - self.assertRaises(Exception, - lambda: env.stat(db1)) - - -class DropTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_empty(self): - _, env = testlib.temp_env() - db1 = env.open_db(B('db1')) - txn = env.begin(write=True) - txn.put(B('a'), B('a'), db=db1) - assert txn.get(B('a'), db=db1) == B('a') - txn.drop(db1, False) - assert txn.get(B('a')) is None - txn.drop(db1, False) # should succeed. - assert txn.get(B('a')) is None - - def test_delete(self): - _, env = testlib.temp_env() - db1 = env.open_db(B('db1')) - txn = env.begin(write=True) - txn.put(B('a'), B('a'), db=db1) - txn.drop(db1) - self.assertRaises(lmdb.InvalidParameterError, - lambda: txn.get(B('a'), db=db1)) - self.assertRaises(lmdb.InvalidParameterError, - lambda: txn.drop(db1)) - - -class CommitTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_bad_txn(self): - _, env = testlib.temp_env() - txn = env.begin() - txn.abort() - self.assertRaises(Exception, - lambda: txn.commit()) - - def test_bad_env(self): - _, env = testlib.temp_env() - txn = env.begin() - env.close() - self.assertRaises(Exception, - lambda: txn.commit()) - - def test_commit_ro(self): - _, env = testlib.temp_env() - txn = env.begin() - txn.commit() - self.assertRaises(Exception, - lambda: txn.commit()) - - def test_commit_rw(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - assert txn.put(B('a'), B('a')) - txn.commit() - self.assertRaises(Exception, - lambda: txn.commit()) - txn = env.begin() - assert txn.get(B('a')) == B('a') - txn.abort() - - -class AbortTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_abort_ro(self): - _, env = testlib.temp_env() - txn = env.begin() - assert txn.get(B('a')) is None - txn.abort() - self.assertRaises(Exception, - lambda: txn.get(B('a'))) - env.close() - self.assertRaises(Exception, - lambda: txn.get(B('a'))) - - def test_abort_rw(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - assert txn.put(B('a'), B('a')) - txn.abort() - txn = env.begin() - assert txn.get(B('a')) is None - - -class GetTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_bad_txn(self): - _, env = testlib.temp_env() - txn = env.begin() - txn.abort() - self.assertRaises(Exception, - lambda: txn.get(B('a'))) - - def test_bad_env(self): - _, env = testlib.temp_env() - txn = env.begin() - env.close() - self.assertRaises(Exception, - lambda: txn.get(B('a'))) - - def test_missing(self): - _, env = testlib.temp_env() - txn = env.begin() - assert txn.get(B('a')) is None - assert txn.get(B('a'), default='default') is 'default' - - def test_empty_key(self): - _, env = testlib.temp_env() - txn = env.begin() - self.assertRaises(lmdb.BadValsizeError, - lambda: txn.get(B(''))) - - def test_db(self): - _, env = testlib.temp_env() - maindb = env.open_db(None) - db1 = env.open_db(B('db1')) - - txn = env.begin() - assert txn.get(B('a'), db=db1) is None - txn.abort() - - txn = env.begin(write=True) - txn.put(B('a'), B('a'), db=db1) - txn.commit() - - txn = env.begin() - assert txn.get(B('a')) is None - txn.abort() - - txn = env.begin(db=db1) - assert txn.get(B('a')) == B('a') - assert txn.get(B('a'), db=maindb) is None - - def test_buffers_no(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - assert txn.put(B('a'), B('a')) - assert type(txn.get(B('a'))) is BytesType - - def test_buffers_yes(self): - _, env = testlib.temp_env() - txn = env.begin(write=True, buffers=True) - assert txn.put(B('a'), B('a')) - assert type(txn.get(B('a'))) is not BytesType - - def test_dupsort(self): - _, env = testlib.temp_env() - db1 = env.open_db(B('db1'), dupsort=True) - txn = env.begin(write=True, db=db1) - assert txn.put(B('a'), B('a')) - assert txn.put(B('a'), B('b')) - assert txn.get(B('a')) == B('a') - - -class PutTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_bad_txn(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - txn.abort() - self.assertRaises(Exception, - lambda: txn.put(B('a'), B('a'))) - - def test_bad_env(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - env.close() - self.assertRaises(Exception, - lambda: txn.put(B('a'), B('a'))) - - def test_ro_txn(self): - _, env = testlib.temp_env() - txn = env.begin() - self.assertRaises(lmdb.ReadonlyError, - lambda: txn.put(B('a'), B('a'))) - - def test_empty_key_value(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - self.assertRaises(lmdb.BadValsizeError, - lambda: txn.put(B(''), B('a'))) - - def test_dupsort(self): - _, env = testlib.temp_env() - - def test_dupdata_no_dupsort(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - assert txn.put(B('a'), B('a'), dupdata=True) - assert txn.put(B('a'), B('b'), dupdata=True) - txn.get(B('a')) - - -class ReplaceTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_bad_txn(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - txn.abort() - self.assertRaises(Exception, - lambda: txn.replace(B('a'), B('a'))) - - def test_bad_env(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - env.close() - self.assertRaises(Exception, - lambda: txn.replace(B('a'), B('a'))) - - def test_ro_txn(self): - _, env = testlib.temp_env() - txn = env.begin() - self.assertRaises(lmdb.ReadonlyError, - lambda: txn.replace(B('a'), B('a'))) - - def test_empty_key_value(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - self.assertRaises(lmdb.BadValsizeError, - lambda: txn.replace(B(''), B('a'))) - - def test_dupsort_noexist(self): - _, env = testlib.temp_env() - db = env.open_db(B('db1'), dupsort=True) - txn = env.begin(write=True, db=db) - assert None == txn.replace(B('a'), B('x')) - assert B('x') == txn.replace(B('a'), B('y')) - assert B('y') == txn.replace(B('a'), B('z')) - cur = txn.cursor() - assert cur.set_key(B('a')) - assert [B('z')] == list(cur.iternext_dup()) - - def test_dupdata_no_dupsort(self): - _, env = testlib.temp_env() - txn = env.begin(write=True) - assert txn.put(B('a'), B('a'), dupdata=True) - assert txn.put(B('a'), B('b'), dupdata=True) - txn.get(B('a')) - - -class LeakTest(unittest.TestCase): - def tearDown(self): - testlib.cleanup() - - def test_open_close(self): - temp_dir = testlib.temp_dir() - env = lmdb.open(temp_dir) - with env.begin() as txn: - pass - env.close() - r1 = weakref.ref(env) - r2 = weakref.ref(txn) - env = None - txn = None - testlib.debug_collect() - assert r1() is None - assert r2() is None - - -if __name__ == '__main__': - unittest.main() diff -Nru stb-tester-30-5-gbefe47c/vendor/py-lmdb/.travis.yml stb-tester-31/vendor/py-lmdb/.travis.yml --- stb-tester-30-5-gbefe47c/vendor/py-lmdb/.travis.yml 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/vendor/py-lmdb/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -language: c -install: -setup: -script: - - "sudo ./misc/runtests-travisci.sh" - -notifications: - email: false diff -Nru stb-tester-30-5-gbefe47c/VERSION stb-tester-31/VERSION --- stb-tester-30-5-gbefe47c/VERSION 2019-03-04 16:44:25.000000000 +0000 +++ stb-tester-31/VERSION 2019-09-18 14:04:32.000000000 +0000 @@ -1 +1 @@ -30-5-gbefe47c +31