diff -Nru skytools-2.1.12/debian/changelog skytools-2.1.12/debian/changelog --- skytools-2.1.12/debian/changelog 2011-11-05 19:37:47.000000000 +0000 +++ skytools-2.1.12/debian/changelog 2011-12-16 10:51:18.000000000 +0000 @@ -1,3 +1,23 @@ +skytools (2.1.12-6) unstable; urgency=low + + * 2.1 is on community support and doesn't release anymore. Import + changes from tag skytools_2_1_12 to branch skytools_2_1_stable from + upstream git. + + Bugfix psycopgwrapper connect_database() to open a cursor before using + it + + Synch walmgr changes from 3.0 branch. + + Add support for detecting stale locks and releasing them instead of + aborting + + Move the pg_stop_backup() into a finally: block. + + londiste add-table: make trigger check sql 9.1-compatible + + pgq.insert_event: Fix rare crash + + pgq.triggers: Fix potential rare crash. + + pgq: fix unnecessary failures in regtests + * Remove \i txid.sql from regression tests so they run through. + Additionally, the server config needs bytea_output = 'escape'. + + -- Christoph Berg Fri, 16 Dec 2011 11:23:55 +0100 + skytools (2.1.12-5) unstable; urgency=low * Fix 9.1 build. Closes: #646666 diff -Nru skytools-2.1.12/debian/patches/missing_includes.diff skytools-2.1.12/debian/patches/missing_includes.diff --- skytools-2.1.12/debian/patches/missing_includes.diff 2011-10-07 21:13:54.000000000 +0000 +++ skytools-2.1.12/debian/patches/missing_includes.diff 2011-12-16 10:23:37.000000000 +0000 @@ -1,6 +1,6 @@ ---- skytools-2.1.8.orig/sql/pgq/triggers/common.c -+++ skytools-2.1.8/sql/pgq/triggers/common.c -@@ -30,6 +30,9 @@ +--- a/sql/pgq/triggers/common.c ++++ b/sql/pgq/triggers/common.c +@@ -31,6 +31,9 @@ #include "common.h" #include "stringutil.h" @@ -10,9 +10,9 @@ /* * Module tag */ ---- skytools-2.1.8.orig/sql/logtriga/logtriga.c -+++ skytools-2.1.8/sql/logtriga/logtriga.c -@@ -21,6 +21,8 @@ +--- a/sql/logtriga/logtriga.c ++++ b/sql/logtriga/logtriga.c +@@ -22,6 +22,8 @@ #include "textbuf.h" @@ -21,4 +21,3 @@ PG_FUNCTION_INFO_V1(logtriga); Datum logtriga(PG_FUNCTION_ARGS); - diff -Nru skytools-2.1.12/debian/patches/regression-fixes skytools-2.1.12/debian/patches/regression-fixes --- skytools-2.1.12/debian/patches/regression-fixes 1970-01-01 00:00:00.000000000 +0000 +++ skytools-2.1.12/debian/patches/regression-fixes 2011-12-16 10:46:26.000000000 +0000 @@ -0,0 +1,20 @@ +--- a/sql/londiste/sql/londiste_install.sql ++++ b/sql/londiste/sql/londiste_install.sql +@@ -1,6 +1,6 @@ + \set ECHO off + set log_error_verbosity = 'terse'; +-\i ../txid/txid.sql ++--\i ../txid/txid.sql + \i ../pgq/pgq.sql + \i ../logtriga/logtriga.sql + \i londiste.sql +--- a/sql/pgq/sql/pgq_init.sql ++++ b/sql/pgq/sql/pgq_init.sql +@@ -1,6 +1,6 @@ + + \set ECHO none +-\i ../txid/txid.sql ++--\i ../txid/txid.sql + \i pgq.sql + + \set ECHO all diff -Nru skytools-2.1.12/debian/patches/series skytools-2.1.12/debian/patches/series --- skytools-2.1.12/debian/patches/series 2011-10-07 21:13:54.000000000 +0000 +++ skytools-2.1.12/debian/patches/series 2011-12-16 10:45:35.000000000 +0000 @@ -1 +1,3 @@ +skytools.git missing_includes.diff +regression-fixes diff -Nru skytools-2.1.12/debian/patches/skytools.git skytools-2.1.12/debian/patches/skytools.git --- skytools-2.1.12/debian/patches/skytools.git 1970-01-01 00:00:00.000000000 +0000 +++ skytools-2.1.12/debian/patches/skytools.git 2011-12-16 10:23:28.000000000 +0000 @@ -0,0 +1,524 @@ +2.1 is on community support and doesn't release anymore. This patch tracks the +changes from tag skytools_2_1_12 to branch skytools_2_1_stable. + +https://github.com/markokr/skytools/ + +--- a/Makefile ++++ b/Makefile +@@ -92,6 +92,12 @@ + yada rebuild + debuild -uc -us -b + ++deb91: ++ ./configure --with-pgconfig=/usr/lib/postgresql/9.1/bin/pg_config --with-python=$(PYTHON) ++ sed -e s/PGVER/9.1/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages ++ yada rebuild ++ debuild -uc -us -b ++ + tgz: config.mak clean + $(MAKE) -C doc man + $(PYTHON) setup.py sdist -t source.cfg -m source.list +--- a/doc/walmgr.txt ++++ b/doc/walmgr.txt +@@ -209,6 +209,9 @@ + How ofter to run periodic command script. In seconds, and only evaluated + at log switch times. + ++==== hot_standby === ++Boolean. If set to true, walmgr setup will set wal_level to hot_standby (9.0 and newer). ++ + === Sample master.ini === + + [wal-master] +@@ -273,6 +276,19 @@ + Script to execute before rotating away the oldest backup. If it fails + backups will not be rotated. + ++ ++==== slave_pg_xlog ==== ++Set slave_pg_xlog to the directory on the slave where pg_xlog files get ++written to. On a restore to the slave walmgr.py will ++create a symbolic link from data/pg_xlog to this location. ++ ++ ++==== backup_datadir ==== ++Set backup_datadir to 'no' to prevent walmgr.py from making a backup ++of the data directory when restoring to the slave. This defaults to ++'yes' ++ ++ + === Sample slave.ini === + + [wal-slave] +@@ -285,4 +301,4 @@ + partial_wals = %(slave)s/logs.partial + full_backup = %(slave)s/data.master + keep_backups = 5 +- ++ backup_datadir = yes +--- a/python/conf/wal-master.ini ++++ b/python/conf/wal-master.ini +@@ -1,16 +1,18 @@ + [wal-master] +-job_name = servername_walgmr_master ++job_name = servername_walmgr_master + logfile = /var/lib/postgresql/log/wal-master.log ++pidfile = /var/lib/postgresql/pid/wal-master.pid + use_skylog = 1 + + master_db = dbname=template1 + master_data = /var/lib/postgresql/8.3/main + master_config = /etc/postgresql/8.3/main/postgresql.conf ++ + # set this only if you can afford database restarts during setup and stop. +-#master_restart_cmd = /etc/init.d/postgresql-8.3 restart ++#master_restart_cmd = /etc/init.d/postgresql-8.3 restart + +-slave = slave:/var/lib/postgresql/walshipping + slave_config = /var/lib/postgresql/conf/wal-slave.ini ++slave = slave:/var/lib/postgresql/walshipping + + completed_wals = %(slave)s/logs.complete + partial_wals = %(slave)s/logs.partial +@@ -21,8 +23,15 @@ + loop_delay = 10.0 + # use record based shipping available since 8.2 + use_xlog_functions = 0 +-# pass -z flag to rsync +-compression = 0 ++ ++# pass -z to rsync, useful on low bandwidth links ++compression = 0 ++ ++# keep symlinks for pg_xlog and pg_log ++keep_symlinks = 1 ++ ++# tell walmgr to set wal_level to hot_standby during setup ++#hot_standby = 1 + + # periodic sync + #command_interval = 600 +--- a/python/conf/wal-slave.ini ++++ b/python/conf/wal-slave.ini +@@ -9,11 +9,20 @@ + slave_start_cmd = /etc/init.d/postgresql-8.3 start + slave_config_dir = /etc/postgresql/8.3/main + ++# alternative pg_xlog directory for slave, symlinked to pg_xlog on restore ++#slave_pg_xlog = /vol2/pg_xlog ++ + slave = /var/lib/postgresql/walshipping + completed_wals = %(slave)s/logs.complete + partial_wals = %(slave)s/logs.partial + full_backup = %(slave)s/data.master + config_backup = %(slave)s/config.backup + ++backup_datadir = yes + keep_backups = 0 + archive_command = ++ ++# primary database connect string for hot standby -- enabling ++# this will cause the slave to be started in hot standby mode. ++#primary_conninfo = host=master port=5432 user=postgres ++ +--- a/python/londiste/setup.py ++++ b/python/londiste/setup.py +@@ -278,11 +278,32 @@ + self.provider_notify_change() + + def provider_add_table(self, tbl): ++ ++ src_db = self.get_database('provider_db') ++ src_curs = src_db.cursor() ++ pg_vers = src_curs.connection.server_version ++ + q = "select londiste.provider_add_table(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, tbl]) + + # detect dangerous triggers +- q = """ ++ if pg_vers >= 90100: ++ q = """ ++ select tg.trigger_name ++ from londiste.provider_table tbl, ++ information_schema.triggers tg ++ where tbl.queue_name = %s ++ and tbl.table_name = %s ++ and tg.event_object_schema = %s ++ and tg.event_object_table = %s ++ and tg.action_timing = 'AFTER' ++ and tg.trigger_name != tbl.trigger_name ++ and tg.trigger_name < tbl.trigger_name ++ and substring(tg.trigger_name from 1 for 10) != '_londiste_' ++ and substring(tg.trigger_name from char_length(tg.trigger_name) - 6) != '_logger' ++ """ ++ else: ++ q = """ + select tg.trigger_name + from londiste.provider_table tbl, + information_schema.triggers tg +@@ -296,9 +317,8 @@ + and substring(tg.trigger_name from 1 for 10) != '_londiste_' + and substring(tg.trigger_name from char_length(tg.trigger_name) - 6) != '_logger' + """ ++ + sname, tname = skytools.fq_name_parts(tbl) +- src_db = self.get_database('provider_db') +- src_curs = src_db.cursor() + src_curs.execute(q, [self.pgq_queue_name, tbl, sname, tname]) + for r in src_curs.fetchall(): + self.log.warning("Table %s has AFTER trigger '%s' which runs before Londiste trigger. "\ +--- a/python/skytools/psycopgwrapper.py ++++ b/python/skytools/psycopgwrapper.py +@@ -99,6 +99,7 @@ + if not hasattr(db, 'server_version'): + iso = db.isolation_level + db.set_isolation_level(0) ++ curs = db.cursor() + curs.execute('show server_version_num') + db.server_version = int(curs.fetchone()[0]) + db.set_isolation_level(iso) +--- a/python/walmgr.py ++++ b/python/walmgr.py +@@ -389,7 +389,7 @@ + self.log.debug("Execute rsync cmd: '%s'" % (cmd)) + if self.not_really: + return 0 +- res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) ++ res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) + if res == 24: + self.log.info("Some files vanished, but thats OK") + res = 0 +@@ -444,15 +444,20 @@ + self.exec_big_rsync(cmdline + [ source_dir, dst_loc ]) + + +- def exec_cmd(self, cmdline): ++ def exec_cmd(self, cmdline,allow_error=False): + cmd = "' '".join(cmdline) + self.log.debug("Execute cmd: '%s'" % (cmd)) + if self.not_really: + return +- res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) +- if res != 0: ++ #res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) ++ process = subprocess.Popen(cmdline,stdout=subprocess.PIPE) ++ output=process.communicate() ++ res = process.returncode ++ ++ if res != 0 and not allow_error: + self.log.fatal("exec failed, res=%d (%s)" % (res, repr(cmdline))) + sys.exit(1) ++ return (res,output[0]) + + def exec_system(self, cmdline): + self.log.debug("Execute cmd: '%s'" % (cmdline)) +@@ -561,26 +566,24 @@ + + else: + # disable archiving ++ cf_params = dict() ++ ++ if can_restart: ++ # can restart, disable archive mode and set wal_level to minimal + +- cf_params = { +- "archive_mode": "off", +- "wal_level": "minimal", +- "archive_command": "" +- } +- +- if curr_archive_mode is None: +- self.log.info("archive_mode not set") +- del(cf_params['archive_mode']) +- +- if curr_wal_level is None: +- self.log.info("wal_level not set") +- del(cf_params['wal_level']) ++ cf_params['archive_command'] = '' + +- if not can_restart: ++ if curr_archive_mode: ++ cf_params['archive_mode'] = 'off' ++ if curr_wal_level: ++ cf_params['wal_level'] = 'minimal' ++ cf_params['max_wal_senders'] = '0' ++ else: + # not possible to change archive_mode or wal_level (requires restart), + # so we just set the archive_command to /bin/true to avoid WAL pileup. + self.log.warning("database must be restarted to disable archiving") + self.log.info("Setting archive_command to /bin/true to avoid WAL pileup") ++ + cf_params['archive_command'] = '/bin/true' + + self.log.debug("modifying configuration: %s" % cf_params) +@@ -623,7 +626,7 @@ + raise Exception("invalid value for 'slave' in %s" % self.cfgfile) + return host + +- def remote_walmgr(self, command, stdin_disabled = True): ++ def remote_walmgr(self, command, stdin_disabled = True,allow_error=False): + """Pass a command to slave WalManager""" + + sshopt = "-T" +@@ -645,7 +648,7 @@ + if self.not_really: + self.log.info("remote_walmgr: %s" % command) + else: +- self.exec_cmd(cmdline) ++ return self.exec_cmd(cmdline,allow_error) + + def walmgr_setup(self): + if self.wtype == MASTER: +@@ -729,7 +732,7 @@ + 5. Release backup lock + """ + +- self.remote_walmgr("xlock") ++ self.remote_xlock() + errors = False + + try: +@@ -805,11 +808,11 @@ + except Exception, e: + self.log.error(e) + errors = True +- +- try: +- self.pg_stop_backup() +- except: +- pass ++ finally: ++ try: ++ self.pg_stop_backup() ++ except: ++ pass + + try: + self.remote_walmgr("xrelease") +@@ -1207,6 +1210,13 @@ + srcfile = os.path.join(srcdir, srcname) + partfile = os.path.join(partdir, srcname) + ++ # if we are using streaming replication, exit immediately ++ # if the srcfile is not here yet ++ primary_conninfo = self.cf.get("primary_conninfo", "") ++ if primary_conninfo and not os.path.isfile(srcfile): ++ self.log.info("%s: not found (ignored)" % srcname) ++ sys.exit(1) ++ + # assume that postgres has processed the WAL file and is + # asking for next - hence work not in progress anymore + if os.path.isfile(prgrfile): +@@ -1446,13 +1456,14 @@ + self.log.debug('using pg_controldata to determine restart points') + restore_command = 'xrestore %f "%p"' + +- conf = """ +-restore_command = '%s %s %s' +-#recovery_target_time= +-#recovery_target_xid= +-#recovery_target_inclusive=true +-#recovery_target_timeline= +-""" % (self.script, cf_file, restore_command) ++ conf = "restore_command = '%s %s %s'\n" % (self.script, cf_file, restore_command) ++ ++ # do we have streaming replication (hot standby) ++ primary_conninfo = self.cf.get("primary_conninfo", "") ++ if primary_conninfo: ++ conf += "standby_mode = 'on'\n" ++ conf += "trigger_file = '%s'\n" % os.path.join(self.cf.get("completed_wals"), "STOP") ++ conf += "primary_conninfo = '%s'\n" % primary_conninfo + + self.log.info("Write %s" % rconf) + if self.not_really: +@@ -1464,8 +1475,7 @@ + + # remove stopfile on slave + if self.wtype == SLAVE: +- srcdir = self.cf.get("completed_wals") +- stopfile = os.path.join(srcdir, "STOP") ++ stopfile = os.path.join(self.cf.get("completed_wals"), "STOP") + if os.path.isfile(stopfile): + self.log.info("Removing stopfile: "+stopfile) + if not self.not_really: +@@ -1577,10 +1587,17 @@ + lockfile = os.path.join(srcdir, "BACKUPLOCK") + if os.path.isfile(lockfile): + self.log.warning("Somebody already has the backup lock.") ++ lockfilehandle = open(lockfile,"r") ++ pidstring = lockfilehandle.read(); ++ try: ++ pid = int(pidstring) ++ print("%d",pid) ++ except ValueError: ++ self.log.error("lock file does not contain a pid:" + pidstring) + return 1 + + if not self.not_really: +- open(lockfile, "w").write("1") ++ open(lockfile, "w").write(self.args[0]) + self.log.info("Backup lock obtained.") + return 0 + +@@ -1773,6 +1790,22 @@ + os.remove(full) + cur_last = fname + return cur_last ++ def remote_xlock(self): ++ ret = self.remote_walmgr("xlock " + str(os.getpid()),allow_error=True) ++ if ret[0] != 0: ++ # lock failed. ++ try: ++ lock_pid = int(ret[1]) ++ if os.kill(lock_pid,0): ++ #process exists. ++ self.log.error("lock already obtained") ++ else: ++ self.remote_walmgr("xrelease") ++ ret = self.remote_walmgr("xlock " + pid(),allow_error=True) ++ if ret[0] != 0: ++ self.log.error("unable to obtain lock") ++ except ValueError: ++ self.log.error("error obtaining lock") + + if __name__ == "__main__": + script = WalMgr(sys.argv[1:]) +--- a/sql/pgq/expected/logutriga.out ++++ b/sql/pgq/expected/logutriga.out +@@ -1,3 +1,4 @@ ++drop function pgq.insert_event(text, text, text, text, text, text, text); + create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text) + returns bigint as $$ + begin +--- a/sql/pgq/expected/pgq_core.out ++++ b/sql/pgq/expected/pgq_core.out +@@ -80,10 +80,10 @@ + 1 + (1 row) + +-select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); +- queue_name | consumer_name | prev_tick_id | tick_id | lag +-------------+---------------+--------------+---------+------------- +- myqueue | consumer | 1 | 2 | @ 0.00 secs ++select queue_name, consumer_name, prev_tick_id, tick_id, lag < '1 second' from pgq.get_batch_info(1); ++ queue_name | consumer_name | prev_tick_id | tick_id | ?column? ++------------+---------------+--------------+---------+---------- ++ myqueue | consumer | 1 | 2 | t + (1 row) + + select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, +--- a/sql/pgq/lowlevel/insert_event.c ++++ b/sql/pgq/lowlevel/insert_event.c +@@ -152,12 +152,17 @@ + entry = hash_search(insert_cache, &queue_id, HASH_ENTER, &did_exist); + if (did_exist) + { +- if (state->cur_table == entry->cur_table) ++ if (entry->plan && state->cur_table == entry->cur_table) + return entry->plan; +- SPI_freeplan(entry->plan); ++ if (entry->plan) ++ SPI_freeplan(entry->plan); + } + entry->cur_table = state->cur_table; ++ entry->plan = NULL; ++ ++ /* this can fail, struct must be valid before */ + entry->plan = make_plan(state); ++ + return entry->plan; + } + +--- a/sql/pgq/sql/pgq_core.sql ++++ b/sql/pgq/sql/pgq_core.sql +@@ -15,7 +15,7 @@ + select pgq.next_batch('myqueue', 'consumer'); + select pgq.next_batch('myqueue', 'consumer'); + +-select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); ++select queue_name, consumer_name, prev_tick_id, tick_id, lag < '1 second' from pgq.get_batch_info(1); + + select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, + queue_switch_time <= now() as switch_time_exists, +--- a/sql/pgq/triggers/common.c ++++ b/sql/pgq/triggers/common.c +@@ -212,9 +212,6 @@ + bool isnull; + int res, i, attno; + +- /* allow reset ASAP, but ignore it in this call */ +- info->invalid = false; +- + /* load pkeys */ + values[0] = ObjectIdGetDatum(rel->rd_id); + res = SPI_execute_plan(pkey_plan, values, NULL, false, 0); +@@ -245,11 +242,23 @@ + } + + static void +-free_info(struct PgqTableInfo *info) ++clean_info(struct PgqTableInfo *info, bool found) + { +- pfree(info->table_name); +- pfree(info->pkey_attno); +- pfree((void *)info->pkey_list); ++ if (!found) ++ goto uninitialized; ++ ++ if (info->table_name) ++ pfree(info->table_name); ++ if (info->pkey_attno) ++ pfree(info->pkey_attno); ++ if (info->pkey_list) ++ pfree((void *)info->pkey_list); ++ ++uninitialized: ++ info->table_name = NULL; ++ info->pkey_attno = NULL; ++ info->pkey_list = NULL; ++ info->n_pkeys = 0; + } + + /* +@@ -281,9 +290,27 @@ + + entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, &found); + if (!found || entry->invalid) { +- if (found) +- free_info(entry); ++ clean_info(entry, found); ++ ++ /* ++ * During fill_tbl_info() 2 events can happen: ++ * - table info reset ++ * - exception ++ * To survive both, always clean struct and tag ++ * as invalid but differently from reset. ++ */ ++ entry->invalid = 2; ++ ++ /* find info */ + fill_tbl_info(rel, entry); ++ ++ /* ++ * If no reset happened, it's valid. Actual reset ++ * is postponed to next call. ++ */ ++ if (entry->invalid == 2) ++ entry->invalid = false; ++ + } + + return entry; +--- a/sql/pgq/triggers/common.h ++++ b/sql/pgq/triggers/common.h +@@ -36,7 +36,7 @@ + const char *pkey_list; /* pk column name list */ + int *pkey_attno; /* pk column positions */ + char *table_name; /* schema-quelified table name */ +- bool invalid; /* set if the info was invalidated */ ++ int invalid; /* set if the info was invalidated */ + }; + + /* common.c */