Software Updater

Merge lp:~jibel/update-manager/AutoUpgradeTester-portlocking into lp:update-manager

  1. AutoUpgradeTester-portlocking
  2. Merge into main
Proposed by Jean-Baptiste Lallement
Status: Merged
Merged at revision: 2326
Proposed branch: lp:~jibel/update-manager/AutoUpgradeTester-portlocking
Merge into: lp:update-manager
Diff against target: 125 lines (+57/-11)
2 files modified
AutoUpgradeTester/UpgradeTestBackendQemu.py (+54/-10)
AutoUpgradeTester/post_upgrade_tests/debconf_test.py (+3/-1)
To merge this branch: bzr merge lp:~jibel/update-manager/AutoUpgradeTester-portlocking
Reviewer Review Type Date Requested Status
Michael Vogt (community) Approve
Review via email: mp+90695@code.launchpad.net

Description of the change

This patch provides the ability to start auto-upgrade-tester runs in parallel.
SSH and VNC ports were not seen as allocated until services were really started by KVM. Now, lock files are used to reserve port for later use by KVM. Tested with Jenkins and 2 executors on each node.

For reference, I used fcntl instead of python-lockfile:
- To avoid a new dependency
- python-lockfile doesn't support locking 2 files from the same process. (http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857)

To post a comment you must log in.
Revision history for this message
Michael Vogt (mvo) wrote :

Thanks a lot for doing this work!

I merged this with some small modifications. E.g. in getFreePort I raise a exception instead of sys.exit() to allow the consumer of the class to decide what to do.

Cheers,
 Michael

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'AutoUpgradeTester/UpgradeTestBackendQemu.py'
--- AutoUpgradeTester/UpgradeTestBackendQemu.py 2012-01-25 22:51:09 +0000
+++ AutoUpgradeTester/UpgradeTestBackendQemu.py 2012-01-30 14:29:04 +0000
@@ -14,6 +14,7 @@
14import tempfile14import tempfile
15import atexit15import atexit
16import apt_pkg16import apt_pkg
17import fcntl
1718
18from DistUpgrade.utils import is_port_already_listening19from DistUpgrade.utils import is_port_already_listening
1920
@@ -108,8 +109,10 @@
108 self.image = os.path.join(imagedir, "test-image.%s" % self.profilename)109 self.image = os.path.join(imagedir, "test-image.%s" % self.profilename)
109 # make ssh login possible (localhost 54321) available110 # make ssh login possible (localhost 54321) available
110 ssh_port = int(self.config.getWithDefault("KVM","SshPort","54321"))111 ssh_port = int(self.config.getWithDefault("KVM","SshPort","54321"))
111 while is_port_already_listening(ssh_port):112 (self.ssh_lock, ssh_port) = self.getFreePort(port_base = ssh_port)
112 ssh_port += 1113 if not self.ssh_lock:
114 print "ERROR: Couldn't allocate SSH port. Exiting!"
115 sys.exit(1)
113 self.ssh_port = str(ssh_port)116 self.ssh_port = str(ssh_port)
114 print "using ssh port: %s" % self.ssh_port117 print "using ssh port: %s" % self.ssh_port
115 self.ssh_hostname = "localhost"118 self.ssh_hostname = "localhost"
@@ -117,12 +120,14 @@
117 self.qemu_options.append("tcp:%s::22" % self.ssh_port)120 self.qemu_options.append("tcp:%s::22" % self.ssh_port)
118 # vnc port/display121 # vnc port/display
119 VNC_BASE_PORT = 5900122 VNC_BASE_PORT = 5900
120 vncport = int(self.config.getWithDefault("KVM","VncNum", "0"))123 vncport = int(self.config.getWithDefault("KVM","VncNum", "0")) + VNC_BASE_PORT
121 while is_port_already_listening(VNC_BASE_PORT + vncport):124 (self.vnc_lock, vncport) = self.getFreePort(port_base = vncport)
122 vncport += 1125 if not self.vnc_lock:
126 print "ERROR: Couldn't allocate VNC port. Exiting!"
127 sys.exit(1)
123 print "using VncNum: %s" % vncport128 print "using VncNum: %s" % vncport
124 self.qemu_options.append("-vnc")129 self.qemu_options.append("-vnc")
125 self.qemu_options.append("localhost:%s" % str(vncport))130 self.qemu_options.append("localhost:%s" % str(vncport - VNC_BASE_PORT))
126131
127 # make the memory configurable132 # make the memory configurable
128 mem = self.config.getWithDefault("KVM","VirtualRam","1536")133 mem = self.config.getWithDefault("KVM","VirtualRam","1536")
@@ -136,8 +141,19 @@
136 # register exit handler to ensure that we quit kvm on exit141 # register exit handler to ensure that we quit kvm on exit
137 atexit.register(self.stop)142 atexit.register(self.stop)
138143
144 def __del__(self):
145 """
146 Destructor
147 Clean-up lockfiles
148 """
149 for lock in (self.ssh_lock, self.vnc_lock):
150 lockpath = lock.name
151 print "Releasing lock: %s" % lockpath
152 lock.close()
153 os.unlink(lockpath)
154
139 def genDiff(self):155 def genDiff(self):
140 """ 156 """
141 generate a diff that compares a fresh install to a upgrade.157 generate a diff that compares a fresh install to a upgrade.
142 ideally that should be empty158 ideally that should be empty
143 Ensure that we always run this *after* the regular upgrade was159 Ensure that we always run this *after* the regular upgrade was
@@ -519,14 +535,42 @@
519 print "Shuting down the VM"535 print "Shuting down the VM"
520 self.stop()536 self.stop()
521 return (ret == 0)537 return (ret == 0)
522 538
539 def getFreePort(self, port_base=1025, prefix='auto-upgrade-tester'):
540 """ Find a free port and lock it when found
541 :param port_base: Base port number.
542 :param prefix: Prefix name for the lock
543 :return: (lockfile, portnumber)
544 """
545
546 lockdir = '/var/lock'
547
548 for port_inc in range(0, 100):
549 port_num = port_base + port_inc
550 if is_port_already_listening(port_num):
551 print "Port %d already in use. Skipping!" % port_num
552 continue
553
554 lockfilepath = os.path.join(lockdir, '%s.%d.lock' % (prefix, port_num))
555 if not os.path.exists(lockfilepath):
556 open(lockfilepath, 'w').close()
557 lock = open(lockfilepath, 'r+')
558 try:
559 fcntl.flock(lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
560 return (lock, port_num)
561 except IOError:
562 print "Port %d already locked. Skipping!" % port_num
563 lock.close()
564
565 print "No free port found. Aborting!"
566 return (None, None)
523567
524if __name__ == "__main__":568if __name__ == "__main__":
525 569
526 # FIXME: very rough proof of conecpt, unify with the chroot570 # FIXME: very rough proof of conecpt, unify with the chroot
527 # and automatic-upgrade code571 # and automatic-upgrade code
528 # see also /usr/sbin/qemu-make-debian-root572 # see also /usr/sbin/qemu-make-debian-root
529 573
530 qemu = UpgradeTestBackendQemu(sys.argv[1],".")574 qemu = UpgradeTestBackendQemu(sys.argv[1],".")
531 #qemu.bootstrap()575 #qemu.bootstrap()
532 #qemu.start()576 #qemu.start()
533577
=== modified file 'AutoUpgradeTester/post_upgrade_tests/debconf_test.py'
--- AutoUpgradeTester/post_upgrade_tests/debconf_test.py 2012-01-18 14:45:04 +0000
+++ AutoUpgradeTester/post_upgrade_tests/debconf_test.py 2012-01-30 14:29:04 +0000
@@ -12,7 +12,9 @@
1212
13# Prompts in this list won't generate a test failure13# Prompts in this list won't generate a test failure
14# i.e WHITELIST = ['libraries/restart-without-asking']14# i.e WHITELIST = ['libraries/restart-without-asking']
15WHITELIST = []15WHITELIST = [
16 'glibc/restart-services',
17 'libraries/restart-without-asking' ]
1618
17def run_test(logfile, resultdir):19def run_test(logfile, resultdir):
18 """ Run the test and slice debconf log20 """ Run the test and slice debconf log

Subscribers

People subscribed via source and target branches

to status/vote changes: