diff -Nru python-whisper-0.9.10/PKG-INFO python-whisper-0.9.12/PKG-INFO --- python-whisper-0.9.10/PKG-INFO 2012-05-31 20:45:55.000000000 +0000 +++ python-whisper-0.9.12/PKG-INFO 2013-08-21 16:57:28.000000000 +0000 @@ -1,8 +1,8 @@ Metadata-Version: 1.0 Name: whisper -Version: 0.9.10 +Version: 0.9.12 Summary: Fixed size round-robin style database -Home-page: https://launchpad.net/graphite +Home-page: http://graphite-project.github.com/ Author: Chris Davis Author-email: chrismd@gmail.com License: Apache Software License 2.0 diff -Nru python-whisper-0.9.10/bin/rrd2whisper.py python-whisper-0.9.12/bin/rrd2whisper.py --- python-whisper-0.9.10/bin/rrd2whisper.py 2012-05-30 22:13:40.000000000 +0000 +++ python-whisper-0.9.12/bin/rrd2whisper.py 2013-04-02 18:47:39.000000000 +0000 @@ -1,67 +1,137 @@ #!/usr/bin/env python -import sys, os, time -import rrdtool -import whisper -from optparse import OptionParser - -option_parser = OptionParser(usage='''%prog rrd_path''') -option_parser.add_option('--xFilesFactor', default=0.5, type='float') +import os +import sys +import time +import signal +import optparse + +try: + import rrdtool +except ImportError, exc: + raise SystemExit('[ERROR] Missing dependency: %s' % str(exc)) + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') + +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +aggregationMethods = whisper.aggregationMethods + +# RRD doesn't have a 'sum' or 'total' type +aggregationMethods.remove('sum') + +option_parser = optparse.OptionParser(usage='''%prog rrd_path''') +option_parser.add_option( + '--xFilesFactor', + help="The xFilesFactor to use in the output file. " + + "Defaults to the input RRD's xFilesFactor", + default=None, + type='float') +option_parser.add_option( + '--aggregationMethod', + help="The consolidation function to fetch from on input and " + + "aggregationMethod to set on output. One of: %s" % + ', '.join(aggregationMethods), + default='average', + type='string') (options, args) = option_parser.parse_args() if len(args) < 1: - option_parser.print_usage() + option_parser.print_help() sys.exit(1) rrd_path = args[0] -rrd_info = rrdtool.info(rrd_path) +try: + rrd_info = rrdtool.info(rrd_path) +except rrdtool.error, exc: + raise SystemExit('[ERROR] %s' % str(exc)) -seconds_per_point = rrd_info['step'] +seconds_per_pdp = rrd_info['step'] -# First get the max retention - we grab the max of all datasources +# Reconcile old vs new python-rrdtool APIs (yuck) +# leave consistent 'rras' and 'datasources' lists if 'rra' in rrd_info: rras = rrd_info['rra'] else: - rra_count = max([ int(key[4]) for key in rrd_info if key.startswith('rra[') ]) + 1 - rras = [{}] * rra_count - for i in range(rra_count): - rras[i]['pdp_per_row'] = rrd_info['rra[%d].pdp_per_row' % i] - rras[i]['rows'] = rrd_info['rra[%d].rows' % i] - -retention_points = 0 -for rra in rras: - points = rra['pdp_per_row'] * rra['rows'] - if points > retention_points: - retention_points = points + rra_indices = [] + for key in rrd_info: + if key.startswith('rra['): + index = int(key.split('[')[1].split(']')[0]) + rra_indices.append(index) -retention = seconds_per_point * points + rra_count = max(rra_indices) + 1 + rras = [] + for i in range(rra_count): + rra_info = {} + rra_info['pdp_per_row'] = rrd_info['rra[%d].pdp_per_row' % i] + rra_info['rows'] = rrd_info['rra[%d].rows' % i] + rra_info['cf'] = rrd_info['rra[%d].cf' % i] + rra_info['xff'] = rrd_info['rra[%d].xff' % i] + rras.append(rra_info) datasources = [] if 'ds' in rrd_info: datasource_names = rrd_info['ds'].keys() else: - ds_keys = [ key for key in rrd_info if key.startswith('ds[') ] - datasources = list(set( key[3:].split(']')[0] for key in ds_keys )) + ds_keys = [key for key in rrd_info if key.startswith('ds[')] + datasources = list(set(key[3:].split(']')[0] for key in ds_keys)) + +# Grab the archive configuration +relevant_rras = [] +for rra in rras: + if rra['cf'] == options.aggregationMethod.upper(): + relevant_rras.append(rra) + +if not relevant_rras: + err = "[ERROR] Unable to find any RRAs with consolidation function: %s" % \ + options.aggregationMethod.upper() + raise SystemExit(err) + +archives = [] +xFilesFactor = options.xFilesFactor +for rra in relevant_rras: + precision = rra['pdp_per_row'] * seconds_per_pdp + points = rra['rows'] + if not xFilesFactor: + xFilesFactor = rra['xff'] + archives.append((precision, points)) for datasource in datasources: - now = int( time.time() ) - path = rrd_path.replace('.rrd','_%s.wsp' % datasource) - whisper.create(path, [(seconds_per_point,retention_points)], xFilesFactor=options.xFilesFactor) + now = int(time.time()) + path = rrd_path.replace('.rrd', '_%s.wsp' % datasource) + try: + whisper.create(path, archives, xFilesFactor=xFilesFactor) + except whisper.InvalidConfiguration, e: + raise SystemExit('[ERROR] %s' % str(e)) size = os.stat(path).st_size - print 'Created: %s (%d bytes)' % (path,size) + archiveConfig = ','.join(["%d:%d" % ar for ar in archives]) + print "Created: %s (%d bytes) with archives: %s" % (path, size, archiveConfig) - print 'Migrating data' - startTime = str(now - retention) - endTime = str(now) - (time_info,columns,rows) = rrdtool.fetch(rrd_path, 'AVERAGE', '-s', startTime, '-e', endTime) - column_index = list(columns).index(datasource) - rows.pop() #remove the last datapoint because RRD sometimes gives funky values - - values = [row[column_index] for row in rows] - timestamps = list(range(*time_info)) - datapoints = zip(timestamps,values) - datapoints = filter(lambda p: p[1] is not None, datapoints) - print ' migrating %d datapoints...' % len(datapoints) - whisper.update_many(path, datapoints) + print "Migrating data" + archiveNumber = len(archives) - 1 + for precision, points in reversed(archives): + retention = precision * points + endTime = now - now % precision + startTime = endTime - retention + (time_info, columns, rows) = rrdtool.fetch( + rrd_path, + options.aggregationMethod.upper(), + '-r', str(precision), + '-s', str(startTime), + '-e', str(endTime)) + column_index = list(columns).index(datasource) + rows.pop() # remove the last datapoint because RRD sometimes gives funky values + + values = [row[column_index] for row in rows] + timestamps = list(range(*time_info)) + datapoints = zip(timestamps, values) + datapoints = filter(lambda p: p[1] is not None, datapoints) + print ' migrating %d datapoints from archive %d' % (len(datapoints), archiveNumber) + archiveNumber -= 1 + whisper.update_many(path, datapoints) diff -Nru python-whisper-0.9.10/bin/whisper-create.py python-whisper-0.9.12/bin/whisper-create.py --- python-whisper-0.9.10/bin/whisper-create.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-create.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,10 +1,19 @@ #!/usr/bin/env python -import sys, os -import whisper -from optparse import OptionParser +import os +import sys +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') -option_parser = OptionParser( +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +option_parser = optparse.OptionParser( usage='''%prog path timePerPoint:timeToStore [timePerPoint:timeToStore]* timePerPoint and timeToStore specify lengths of time, for example: @@ -30,11 +39,14 @@ archives = [whisper.parseRetentionDef(retentionDef) for retentionDef in args[1:]] -if options.overwrite and os.path.exists(path): - print 'Overwriting existing file: %s' % path - os.unlink(path) - -whisper.create(path, archives, xFilesFactor=options.xFilesFactor, aggregationMethod=options.aggregationMethod) +if os.path.exists(path) and options.overwrite: + print 'Overwriting existing file: %s' % path + os.unlink(path) + +try: + whisper.create(path, archives, xFilesFactor=options.xFilesFactor, aggregationMethod=options.aggregationMethod) +except whisper.WhisperException, exc: + raise SystemExit('[ERROR] %s' % str(exc)) size = os.stat(path).st_size print 'Created: %s (%d bytes)' % (path,size) diff -Nru python-whisper-0.9.10/bin/whisper-dump.py python-whisper-0.9.12/bin/whisper-dump.py --- python-whisper-0.9.10/bin/whisper-dump.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-dump.py 2012-10-09 20:35:34.000000000 +0000 @@ -1,12 +1,20 @@ #!/usr/bin/env python import os -import struct -import whisper import mmap -from optparse import OptionParser +import struct +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') -option_parser = OptionParser(usage='''%prog path''') +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +option_parser = optparse.OptionParser(usage='''%prog path''') (options, args) = option_parser.parse_args() if len(args) != 1: @@ -15,8 +23,8 @@ path = args[0] def mmap_file(filename): - fd = os.open(filename, os.O_RDONLY) - map = mmap.mmap(fd, 0, prot=mmap.PROT_READ) + fd = os.open(filename, os.O_RDONLY) + map = mmap.mmap(fd, os.fstat(fd).st_size, prot=mmap.PROT_READ) os.close(fd) return map @@ -33,7 +41,7 @@ try: (offset, secondsPerPoint, points) = struct.unpack(whisper.archiveInfoFormat, map[archiveOffset:archiveOffset+whisper.archiveInfoSize]) except: - raise CorruptWhisperFile("Unable to reda archive %d metadata" % i) + raise CorruptWhisperFile("Unable to read archive %d metadata" % i) archiveInfo = { 'offset' : offset, @@ -81,6 +89,9 @@ offset += whisper.pointSize print +if not os.path.exists(path): + raise SystemExit('[ERROR] File "%s" does not exist!' % path) + map = mmap_file(path) header = read_header(map) dump_header(header) diff -Nru python-whisper-0.9.10/bin/whisper-fetch.py python-whisper-0.9.12/bin/whisper-fetch.py --- python-whisper-0.9.10/bin/whisper-fetch.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-fetch.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,13 +1,22 @@ #!/usr/bin/env python -import sys, time -import whisper -from optparse import OptionParser +import sys +import time +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') + +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) now = int( time.time() ) yesterday = now - (60 * 60 * 24) -option_parser = OptionParser(usage='''%prog [options] path''') +option_parser = optparse.OptionParser(usage='''%prog [options] path''') option_parser.add_option('--from', default=yesterday, type='int', dest='_from', help=("Unix epoch time of the beginning of " "your requested interval (default: 24 hours ago)")) @@ -30,7 +39,11 @@ until_time = int( options.until ) -(timeInfo, values) = whisper.fetch(path, from_time, until_time) +try: + (timeInfo, values) = whisper.fetch(path, from_time, until_time) +except whisper.WhisperException, exc: + raise SystemExit('[ERROR] %s' % str(exc)) + (start,end,step) = timeInfo diff -Nru python-whisper-0.9.10/bin/whisper-info.py python-whisper-0.9.12/bin/whisper-info.py --- python-whisper-0.9.10/bin/whisper-info.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-info.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,10 +1,19 @@ #!/usr/bin/env python -import sys, os -import whisper -from optparse import OptionParser +import os +import sys +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') -option_parser = OptionParser(usage='''%prog path [field]''') +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +option_parser = optparse.OptionParser(usage='''%prog path [field]''') (options, args) = option_parser.parse_args() if len(args) < 1: @@ -17,7 +26,11 @@ else: field = None -info = whisper.info(path) +try: + info = whisper.info(path) +except whisper.WhisperException, exc: + raise SystemExit('[ERROR] %s' % str(exc)) + info['fileSize'] = os.stat(path).st_size if field: diff -Nru python-whisper-0.9.10/bin/whisper-merge.py python-whisper-0.9.12/bin/whisper-merge.py --- python-whisper-0.9.10/bin/whisper-merge.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-merge.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,11 +1,19 @@ #!/usr/bin/env python +import os import sys -import whisper +import signal +import optparse -from optparse import OptionParser +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') -option_parser = OptionParser( +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +option_parser = optparse.OptionParser( usage='''%prog [options] from_path to_path''') (options, args) = option_parser.parse_args() @@ -17,4 +25,8 @@ path_from = args[0] path_to = args[1] +for filename in (path_from, path_to): + if not os.path.exists(filename): + raise SystemExit('[ERROR] File "%s" does not exist!' % filename) + whisper.merge(path_from, path_to) diff -Nru python-whisper-0.9.10/bin/whisper-resize.py python-whisper-0.9.12/bin/whisper-resize.py --- python-whisper-0.9.10/bin/whisper-resize.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-resize.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,12 +1,25 @@ #!/usr/bin/env python -import sys, os, time, traceback -import whisper -from optparse import OptionParser +import os +import sys +import math +import time +import bisect +import signal +import optparse +import traceback + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') + +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) now = int(time.time()) -option_parser = OptionParser( +option_parser = optparse.OptionParser( usage='''%prog path timePerPoint:timeToStore [timePerPoint:timeToStore]* timePerPoint and timeToStore specify lengths of time, for example: @@ -33,6 +46,10 @@ option_parser.add_option( '--nobackup', action='store_true', help='Delete the .bak file after successful execution') +option_parser.add_option( + '--aggregate', action='store_true', + help='Try to aggregate the values to fit the new archive better.' + ' Note that this will make things slower and use more memory.') (options, args) = option_parser.parse_args() @@ -41,10 +58,17 @@ sys.exit(1) path = args[0] + +if not os.path.exists(path): + sys.stderr.write("[ERROR] File '%s' does not exist!\n\n" % path) + option_parser.print_usage() + sys.exit(1) + +info = whisper.info(path) + new_archives = [whisper.parseRetentionDef(retentionDef) for retentionDef in args[1:]] -info = whisper.info(path) old_archives = info['archives'] # sort by precision, lowest to highest old_archives.sort(key=lambda a: a['secondsPerPoint'], reverse=True) @@ -80,12 +104,64 @@ size = os.stat(newfile).st_size print 'Created: %s (%d bytes)' % (newfile,size) -print 'Migrating data...' -for archive in old_archives: - timeinfo, values = archive['data'] - datapoints = zip( range(*timeinfo), values ) - datapoints = filter(lambda p: p[1] is not None, datapoints) - whisper.update_many(newfile, datapoints) +if options.aggregate: + # This is where data will be interpolated (best effort) + print 'Migrating data with aggregation...' + all_datapoints = [] + for archive in old_archives: + # Loading all datapoints into memory for fast querying + timeinfo, values = archive['data'] + new_datapoints = zip( range(*timeinfo), values ) + if all_datapoints: + last_timestamp = all_datapoints[-1][0] + slice_end = 0 + for i,(timestamp,value) in enumerate(new_datapoints): + if timestamp > last_timestamp: + slice_end = i + break + all_datapoints += new_datapoints[i:] + else: + all_datapoints += new_datapoints + + oldtimestamps = map( lambda p: p[0], all_datapoints) + oldvalues = map( lambda p: p[1], all_datapoints) + + print "oldtimestamps: %s" % oldtimestamps + # Simply cleaning up some used memory + del all_datapoints + + new_info = whisper.info(newfile) + new_archives = new_info['archives'] + + for archive in new_archives: + step = archive['secondsPerPoint'] + fromTime = now - archive['retention'] + now % step + untilTime = now + now % step + step + print "(%s,%s,%s)" % (fromTime,untilTime, step) + timepoints_to_update = range(fromTime, untilTime, step) + print "timepoints_to_update: %s" % timepoints_to_update + newdatapoints = [] + for tinterval in zip( timepoints_to_update[:-1], timepoints_to_update[1:] ): + # TODO: Setting lo= parameter for 'lefti' based on righti from previous + # iteration. Obviously, this can only be done if + # timepoints_to_update is always updated. Is it? + lefti = bisect.bisect_left(oldtimestamps, tinterval[0]) + righti = bisect.bisect_left(oldtimestamps, tinterval[1], lo=lefti) + newvalues = oldvalues[lefti:righti] + if newvalues: + non_none = filter( lambda x: x is not None, newvalues) + if 1.0*len(non_none)/len(newvalues) >= xff: + newdatapoints.append([tinterval[0], + whisper.aggregate(aggregationMethod, + non_none)]) + whisper.update_many(newfile, newdatapoints) +else: + print 'Migrating data without aggregation...' + for archive in old_archives: + timeinfo, values = archive['data'] + datapoints = zip( range(*timeinfo), values ) + datapoints = filter(lambda p: p[1] is not None, datapoints) + whisper.update_many(newfile, datapoints) if options.newfile is not None: sys.exit(0) diff -Nru python-whisper-0.9.10/bin/whisper-set-aggregation-method.py python-whisper-0.9.12/bin/whisper-set-aggregation-method.py --- python-whisper-0.9.10/bin/whisper-set-aggregation-method.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-set-aggregation-method.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,10 +1,19 @@ #!/usr/bin/env python -import sys, os -import whisper -from optparse import OptionParser +import os +import sys +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') -option_parser = OptionParser( +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +option_parser = optparse.OptionParser( usage='%%prog path <%s>' % '|'.join(whisper.aggregationMethods)) (options, args) = option_parser.parse_args() @@ -16,6 +25,14 @@ path = args[0] aggregationMethod = args[1] -oldAggregationMethod = whisper.setAggregationMethod(path, aggregationMethod) +try: + oldAggregationMethod = whisper.setAggregationMethod(path, aggregationMethod) +except IOError, exc: + sys.stderr.write("[ERROR] File '%s' does not exist!\n\n" % path) + option_parser.print_usage() + sys.exit(1) +except whisper.WhisperException, exc: + raise SystemExit('[ERROR] %s' % str(exc)) + print 'Updated aggregation method: %s (%s -> %s)' % (path,oldAggregationMethod,aggregationMethod) diff -Nru python-whisper-0.9.10/bin/whisper-update.py python-whisper-0.9.12/bin/whisper-update.py --- python-whisper-0.9.10/bin/whisper-update.py 2012-05-08 04:26:31.000000000 +0000 +++ python-whisper-0.9.12/bin/whisper-update.py 2013-08-20 17:56:53.000000000 +0000 @@ -1,12 +1,21 @@ #!/usr/bin/env python -import sys, time -import whisper -from optparse import OptionParser +import sys +import time +import signal +import optparse + +try: + import whisper +except ImportError: + raise SystemExit('[ERROR] Please make sure whisper is installed properly') + +# Ignore SIGPIPE +signal.signal(signal.SIGPIPE, signal.SIG_DFL) now = int( time.time() ) -option_parser = OptionParser( +option_parser = optparse.OptionParser( usage='''%prog [options] path timestamp:value [timestamp:value]*''') (options, args) = option_parser.parse_args() @@ -21,9 +30,11 @@ for point in datapoint_strings] datapoints = [tuple(point.split(':')) for point in datapoint_strings] -if len(datapoints) == 1: - timestamp,value = datapoints[0] - whisper.update(path, value, timestamp) -else: - print datapoints - whisper.update_many(path, datapoints) +try: + if len(datapoints) == 1: + timestamp,value = datapoints[0] + whisper.update(path, value, timestamp) + else: + whisper.update_many(path, datapoints) +except whisper.WhisperException, exc: + raise SystemExit('[ERROR] %s' % str(exc)) diff -Nru python-whisper-0.9.10/debian/changelog python-whisper-0.9.12/debian/changelog --- python-whisper-0.9.10/debian/changelog 2012-06-02 18:05:47.000000000 +0000 +++ python-whisper-0.9.12/debian/changelog 2013-09-01 10:00:21.000000000 +0000 @@ -1,3 +1,16 @@ +python-whisper (0.9.12-1) unstable; urgency=low + + [ Jakub Wilk ] + * Use canonical URIs for Vcs-* fields. + + [ Jonas Genannt ] + * New Upstream Version + * d/control: bumped standards version + * d/control: updated homepage field + * changed from python-support to dh_python2 (Closes: #721128) + + -- Jonas Genannt Sun, 01 Sep 2013 11:54:41 +0200 + python-whisper (0.9.10-1) unstable; urgency=low * New upstream version diff -Nru python-whisper-0.9.10/debian/control python-whisper-0.9.12/debian/control --- python-whisper-0.9.10/debian/control 2012-06-02 18:05:47.000000000 +0000 +++ python-whisper-0.9.12/debian/control 2013-09-01 09:54:09.000000000 +0000 @@ -3,12 +3,11 @@ Priority: optional Maintainer: Debian Python Modules Team Uploaders: Elliot Murphy , Jonas Genannt -Build-Depends: debhelper (>= 8), python-support (>= 0.6.4), python (>= 2.5), perl -XS-Python-Version: >= 2.5 -Standards-Version: 3.9.3 -Homepage: https://launchpad.net/graphite -Vcs-Svn: svn://svn.debian.org/python-modules/packages/python-whisper/trunk/ -Vcs-Browser: http://svn.debian.org/viewsvn/python-modules/packages/python-whisper/trunk/ +Build-Depends: debhelper (>= 8), python (>= 2.6.6-3~), perl +Standards-Version: 3.9.4 +Homepage: https://github.com/graphite-project/whisper +Vcs-Svn: svn://anonscm.debian.org/python-modules/packages/python-whisper/trunk/ +Vcs-Browser: http://anonscm.debian.org/viewvc/python-modules/packages/python-whisper/trunk/ Package: python-whisper Architecture: all diff -Nru python-whisper-0.9.10/debian/patches/series python-whisper-0.9.12/debian/patches/series --- python-whisper-0.9.10/debian/patches/series 2012-06-02 18:05:47.000000000 +0000 +++ python-whisper-0.9.12/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -whisper-dump-typo.patch diff -Nru python-whisper-0.9.10/debian/patches/whisper-dump-typo.patch python-whisper-0.9.12/debian/patches/whisper-dump-typo.patch --- python-whisper-0.9.10/debian/patches/whisper-dump-typo.patch 2012-06-02 18:05:47.000000000 +0000 +++ python-whisper-0.9.12/debian/patches/whisper-dump-typo.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -Description: Fix typo in whisper-dump script -Author: Jonas Genannt -Bug: https://bugs.launchpad.net/graphite/+bug/1007893 - -diff --git a/bin/whisper-dump.py b/bin/whisper-dump.py -index 0a34f82..44c4fc5 100755 ---- a/bin/whisper-dump.py -+++ b/bin/whisper-dump.py -@@ -33,7 +33,7 @@ def read_header(map): - try: - (offset, secondsPerPoint, points) = struct.unpack(whisper.archiveInfoFormat, map[archiveOffset:archiveOffset+whisper.archiveInfoSize]) - except: -- raise CorruptWhisperFile("Unable to reda archive %d metadata" % i) -+ raise CorruptWhisperFile("Unable to read archive %d metadata" % i) - - archiveInfo = { - 'offset' : offset, diff -Nru python-whisper-0.9.10/debian/rules python-whisper-0.9.12/debian/rules --- python-whisper-0.9.10/debian/rules 2012-06-02 16:46:25.000000000 +0000 +++ python-whisper-0.9.12/debian/rules 2013-09-01 09:54:09.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/make -f %: - dh $@ + dh $@ --with python2 override_dh_install: rename 's/\.py//' debian/python-whisper/usr/bin/*.py diff -Nru python-whisper-0.9.10/setup.py python-whisper-0.9.12/setup.py --- python-whisper-0.9.10/setup.py 2012-05-31 20:33:15.000000000 +0000 +++ python-whisper-0.9.12/setup.py 2013-08-21 16:56:08.000000000 +0000 @@ -7,8 +7,8 @@ setup( name='whisper', - version='0.9.10', - url='https://launchpad.net/graphite', + version='0.9.12', + url='http://graphite-project.github.com/', author='Chris Davis', author_email='chrismd@gmail.com', license='Apache Software License 2.0', diff -Nru python-whisper-0.9.10/whisper.py python-whisper-0.9.12/whisper.py --- python-whisper-0.9.10/whisper.py 2012-05-16 04:11:41.000000000 +0000 +++ python-whisper-0.9.12/whisper.py 2013-08-20 17:56:53.000000000 +0000 @@ -32,6 +32,42 @@ except ImportError: CAN_LOCK = False +try: + import ctypes + import ctypes.util + CAN_FALLOCATE = True +except ImportError: + CAN_FALLOCATE = False + +fallocate = None + +if CAN_FALLOCATE: + libc_name = ctypes.util.find_library('c') + libc = ctypes.CDLL(libc_name) + c_off64_t = ctypes.c_int64 + c_off_t = ctypes.c_int + + try: + _fallocate = libc.posix_fallocate64 + _fallocate.restype = ctypes.c_int + _fallocate.argtypes = [ctypes.c_int, c_off64_t, c_off64_t] + except AttributeError, e: + try: + _fallocate = libc.posix_fallocate + _fallocate.restype = ctypes.c_int + _fallocate.argtypes = [ctypes.c_int, c_off_t, c_off_t] + except AttributeError, e: + CAN_FALLOCATE = False + + if CAN_FALLOCATE: + def _py_fallocate(fd, offset, len_): + res = _fallocate(fd.fileno(), offset, len_) + if res != 0: + raise IOError(res, 'fallocate') + fallocate = _py_fallocate + del libc + del libc_name + LOCK = False CACHE_HEADERS = False AUTOFLUSH = False @@ -292,7 +328,7 @@ raise InvalidConfiguration("Lower precision archives must cover " "larger time intervals than higher precision archives " "(archive%d: %s seconds, archive%d: %s seconds)" % - (i, archive[1], i + 1, nextArchive[1])) + (i, retention, i + 1, nextRetention)) archivePoints = archive[1] pointsPerConsolidation = nextArchive[0] / archive[0] @@ -303,7 +339,7 @@ (i + 1, pointsPerConsolidation, i, archivePoints)) -def create(path,archiveList,xFilesFactor=None,aggregationMethod=None,sparse=False): +def create(path,archiveList,xFilesFactor=None,aggregationMethod=None,sparse=False,useFallocate=False): """create(path,archiveList,xFilesFactor=0.5,aggregationMethod='average') path is a string @@ -329,7 +365,7 @@ fcntl.flock( fh.fileno(), fcntl.LOCK_EX ) aggregationType = struct.pack( longFormat, aggregationMethodToType.get(aggregationMethod, 1) ) - oldest = sorted([secondsPerPoint * points for secondsPerPoint,points in archiveList])[-1] + oldest = max([secondsPerPoint * points for secondsPerPoint,points in archiveList]) maxRetention = struct.pack( longFormat, oldest ) xFilesFactor = struct.pack( floatFormat, float(xFilesFactor) ) archiveCount = struct.pack(longFormat, len(archiveList)) @@ -343,12 +379,15 @@ fh.write(archiveInfo) archiveOffsetPointer += (points * pointSize) - if sparse: - fh.seek(archiveOffsetPointer - headerSize - 1) - fh.write("\0") + #If configured to use fallocate and capable of fallocate use that, else + #attempt sparse if configure or zero pre-allocate if sparse isn't configured. + if CAN_FALLOCATE and useFallocate: + remaining = archiveOffsetPointer - headerSize + fallocate(fh, headerSize, remaining) + elif sparse: + fh.seek(archiveOffsetPointer - 1) + fh.write('\x00') else: - # If not creating the file sparsely, then fill the rest of the file with - # zeroes. remaining = archiveOffsetPointer - headerSize chunksize = 16384 zeroes = '\x00' * chunksize @@ -363,7 +402,7 @@ fh.close() -def __aggregate(aggregationMethod, knownValues): +def aggregate(aggregationMethod, knownValues): if aggregationMethod == 'average': return float(sum(knownValues)) / float(len(knownValues)) elif aggregationMethod == 'sum': @@ -437,7 +476,7 @@ knownPercent = float(len(knownValues)) / float(len(neighborValues)) if knownPercent >= xff: #we have enough data to propagate a value! - aggregateValue = __aggregate(aggregationMethod, knownValues) + aggregateValue = aggregate(aggregationMethod, knownValues) myPackedPoint = struct.pack(pointFormat,lowerIntervalStart,aggregateValue) fh.seek(lower['offset']) packedPoint = fh.read(pointSize) @@ -582,12 +621,12 @@ step = archive['secondsPerPoint'] alignedPoints = [ (timestamp - (timestamp % step), value) for (timestamp,value) in points ] + alignedPoints = dict(alignedPoints).items() # Take the last val of duplicates #Create a packed string for each contiguous sequence of points packedStrings = [] previousInterval = None currentString = "" for (interval,value) in alignedPoints: - if interval == previousInterval: continue if (not previousInterval) or (interval == previousInterval + step): currentString += struct.pack(pointFormat,interval,value) previousInterval = interval @@ -661,7 +700,12 @@ path is a string fromTime is an epoch time -untilTime is also an epoch time, but defaults to now +untilTime is also an epoch time, but defaults to now. + +Returns a tuple of (timeInfo, valueList) +where timeInfo is itself a tuple of (fromTime, untilTime, step) + +Returns None if no data can be returned """ fh = open(path,'rb') return file_fetch(fh, fromTime, untilTime) @@ -675,16 +719,25 @@ fromTime = int(fromTime) untilTime = int(untilTime) + # Here we try and be flexible and return as much data as we can. + # If the range of data is from too far in the past or fully in the future, we + # return nothing + if (fromTime > untilTime): + raise InvalidTimeInterval("Invalid time interval: from time '%s' is after until time '%s'" % (fromTime, untilTime)) + oldestTime = now - header['maxRetention'] + # Range is in the future + if fromTime > now: + return None + # Range is beyond retention + if untilTime < oldestTime: + return None + # Range requested is partially beyond retention, adjust if fromTime < oldestTime: fromTime = oldestTime - - if not (fromTime < untilTime): - raise InvalidTimeInterval("Invalid time interval") + # Range is partially in the future, adjust if untilTime > now: untilTime = now - if untilTime < fromTime: - untilTime = now diff = now - fromTime for archive in header['archives']: