diff -Nru debdelta-0.44/contrib/debmirror-delta-security debdelta-0.45/contrib/debmirror-delta-security --- debdelta-0.44/contrib/debmirror-delta-security 2011-08-24 11:31:41.000000000 +0000 +++ debdelta-0.45/contrib/debmirror-delta-security 2011-12-05 17:23:47.000000000 +0000 @@ -95,7 +95,8 @@ # see in /usr/share/debdelta $DEBMIRROR $DEBUG $VERBOSE $secdebmir $DEBMIRROR_TRASH \ --method=$DEBMIRROR_METHOD --nosource -h $sechost \ - -r $release -d ${secstable} --arch=$ARCHc + -r $release -d ${secstable} --arch=$ARCHc \ + $DEBMIRROR_OPTIONS #do create deltas lockfile -r 1 /tmp/$b.lock || exit 1 diff -Nru debdelta-0.44/contrib/debmirror-delta-security_ubuntu.conf debdelta-0.45/contrib/debmirror-delta-security_ubuntu.conf --- debdelta-0.44/contrib/debmirror-delta-security_ubuntu.conf 2011-08-24 11:31:41.000000000 +0000 +++ debdelta-0.45/contrib/debmirror-delta-security_ubuntu.conf 2011-12-05 17:23:47.000000000 +0000 @@ -12,6 +12,7 @@ # to security2 will be generated DEBMIRROR_TRASH="--trash $deltamir/old_debs" DEBMIRROR_METHOD="http" +DEBMIRROR_OPTIONS="--passive" #where the security archive is sechost=security.ubuntu.com diff -Nru debdelta-0.44/debdelta debdelta-0.45/debdelta --- debdelta-0.44/debdelta 2011-08-25 11:05:53.000000000 +0000 +++ debdelta-0.45/debdelta 2011-12-06 16:22:44.000000000 +0000 @@ -169,6 +169,21 @@ DEB_POLICY = ['b','s','e'] DO_PROGRESS = terminalcolumns != None +#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails +#possible values: +# False : do not send them +# True : compute forensic but not send them, just list them +# mail : automatically send by email to default address +# user@domain : automatically send by email to address +# mailto:user@domain : as above +# mutt:user@domain : as above, but use 'mutt', so the user can customize it +# http://domain/cgi : send them automatically thru a CGI script +#Warning: the above is mostly TODO +FORENSIC=False + +#directory tree where forensic info are stored by 'debdeltas' +FORENSICDIR=None + DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') @@ -219,7 +234,7 @@ try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', - 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug', + 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=") ) except getopt.GetoptError,a: sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') @@ -260,6 +275,24 @@ if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) + + elif o == '--forensicdir' : + FORENSICDIR = abspath(expanduser(v)) + if v[-2:] == '//': + FORENSICDIR += '//' + if not os.path.isdir(FORENSICDIR): + sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') + raise SystemExit(3) + + elif o == '--forensic' : + FORENSIC = v + if FORENSIC[:4] == 'http': + try: + import poster + except: + print 'To use the http forensic, you must install the package "python-poster".' + raise SystemExit(3) + elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') @@ -410,7 +443,9 @@ def de_bar(a): if a and a[:2] == './' : a=a[2:] - if a and a[0] == '/' : + elif a == '/.' : + a='' + elif a and a[0] == '/' : a=a[1:] return a @@ -546,7 +581,7 @@ class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. - def __init__(self,s,retriable=False,exitcode=None): + def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) == StringType) self.retriable = retriable if retriable: @@ -559,6 +594,7 @@ else: exitcode = 2 self.exitcode=exitcode + self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') @@ -1116,43 +1152,132 @@ a=a+ ( '%02x' % ord(i) ) return a -def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]): - o.write('Package: '+db['OLD/Package']+'\n') - o.write('Version: '+db['OLD/Version']+'\n') - o.write('Architecture: '+db['OLD/Architecture']+'\n') +def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): + " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" + assert type(diversions) == dict + if type(db) == dict: + for a in sorted(db.keys()): + if a[:3] == 'OLD': + o.write(a[4:]+': '+db[a]+'\n') + else: + for a in sorted(db): + if a[:3] == 'OLD': + o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") - for a in diverted: + for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") - for a in conffiles: + for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") - for a,b in L: - if not os.path.exists(b): - o.write(' NONEXISTENT\n '+b+'\n \n') + for l in sorted(L): + if bytar: + name,mode,tartype,uid,gid,uname,gname,data=l + tmpcopy=None + divert=None + else: + name,divert,tmpcopy=l + if os.path.exists(divert): + fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) + else: + fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' + if tartype == tarfile.REGTYPE: + if tmpcopy and os.path.exists(tmpcopy): + data=hash_to_hex(sha1_hash_file(tmpcopy)) + elif os.path.exists(divert): + data=hash_to_hex(sha1_hash_file(divert)) + if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue - name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b) - if tartype == tarfile.REGTYPE: - data=hash_to_hex(sha1_hash_file(b)) if uname == None: uname=str(uid) if gname == None: gname=str(gid) + name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) - if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles: - o.write(" [conffile]\n") - else: - o.write("\n") - o.write(" "+a+"\n") + if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: + o.write(" [conffile]") + if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: + o.write(" [localpurged]") + if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: + o.write(" [prelink-u failed]") + if divert and not os.path.exists(divert): + o.write(" [missing file %r]" % divert) + if tmpcopy: + o.write(" [prelink-u]") + o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") +def forensic_send(f,forensic=FORENSIC): + " note that f must be a list of lists (or None)" + assert type(f) == list + if not forensic : + if f: + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + return + if not f: + return + if all([(z == None) for z in f]): + print 'Sorry, no forensic logs were generated' + return + if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': + email=EMAIL + if ':' in forensic: + a=forensic.find(':') + email == forensic[a:] + forensic=forensic[:a] + print _("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.") + if forensic in ('mutt','mail'): + raw_input( _('(hit any key)') ) + args=[] + for z in f: + if z: + for j in z: + args+=['-a',j] + subprocess.call(['mutt',email,'-s','delta_failures']+args) + else: + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) + subprocess.call([forensic,'-compose',args]) + return + elif forensic[:4] == 'http': + print _("There were faulty deltas.")+' '+_('Sending logs to server.') + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + #http://atlee.ca/software/poster + import urllib, urllib2, httplib, poster + poster.streaminghttp.register_openers() + datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) + # Create the Request object + request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) + # Actually do the request, and get the response + print ' '+_('Server answers:'),repr(urllib2.urlopen(request).read()) + return + else: + sys.stderr.write(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) + for z in f: + if z: + sys.stderr.write(' '+string.join(z,' ')+'\n') + return + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct @@ -1302,11 +1427,11 @@ #this is not needed in preparing the patch, but may help in forensic conf_files=[] - a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' - if DEBUG and os.path.isfile(a): - #note that filenames have leading / - conf_files=[p for p in open(a).read().split('\n') if p] - del a + z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' + if DEBUG and os.path.isfile(z): + #note that filenames do not have leading / + conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] + del z ### s=patch_check_tmp_space(params,olddeb) @@ -1350,6 +1475,7 @@ elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) + del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 @@ -1399,9 +1525,10 @@ p.close() return s, diverted - localepurged=[] - prelink_u_failed=[] def _symlink_data_tree(pa,TD,diversions,runtime): + localepurged=[] + prelink_u_failed=[] + file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: @@ -1417,8 +1544,8 @@ if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : - a=TD+'OLD/DATA'+orig - d=os.path.dirname(a) + tmpcopy=TD+'OLD/DATA'+orig + d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru @@ -1433,40 +1560,54 @@ prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : - print ' copying/unprelinking ',divert,' to ',a + print ' copying/unprelinking ',divert,' to ', tmpcopy #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 - shutil.copy2(divert,a) - proc=subprocess.Popen(["/usr/sbin/prelink","-u",a],stdin=open(os.devnull),\ + shutil.copy2(divert, tmpcopy) + proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT) out=proc.stdout.read().strip() proc.wait() if proc.returncode: - if not os.path.exists(a): - if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',a,')' - os.symlink(divert, a) + if not os.path.exists(tmpcopy): + if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',tmpcopy,')' + os.symlink(divert, tmpcopy) + prelink_u_failed.append(de_bar(orig)) + unprelink=False elif VERBOSE > 4 : print ' (prelink failed, but file was copied)' - thestat = os.statvfs(a) + thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG: sys.stderr.write('!!'+repr(out)+'\n') elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : sys.stderr.write('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') - prelink_u_failed.append(a) + prelink_u_failed.append(de_bar(orig)) + unprelink=False else: - sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (a,out)) - prelink_u_failed.append(a) + sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (tmpcopy,out)) + prelink_u_failed.append(de_bar(orig)) + unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : print ' symlinking ',divert,' to ',a - os.symlink(divert, a) + os.symlink(divert, tmpcopy) + if unprelink and FORENSIC: + #unfortunately the script will delete the 'tmpcopy', so we hardlink it + z=tempfile.mktemp(prefix=TD) + os.link(tmpcopy,z) + file_triples.append((orig,divert,z)) + else: + file_triples.append((orig,divert,None)) elif not os.path.exists(divert): + file_triples.append((orig,divert,None)) if VERBOSE : print ' Disappeared file? ',divert for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: - localepurged.append(orig) - elif VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig - return s,diverted, prelink_time, prelink_datasize + localepurged.append(de_bar(orig)) + else: + file_triples.append((orig,divert,None)) + if VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig + return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" @@ -1487,8 +1628,14 @@ i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) - control_file_pairs=[] - linked_file_pairs,diverted=[],[] + #initialize, just in case + control_file_triples=[] + file_triples=[] + localepurged=[] + prelink_u_failed=[] + diverted=[] + prelink_time=0 + prelink_datasize=0 ###see into parameters: the patch may need extra info and data @@ -1503,7 +1650,8 @@ elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if olddeb == '/': - linked_file_pairs,diverted,prelink_time,prelink_datasize=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) + file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ + _symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if 'data.tar.bz2' in ar_list_old: @@ -1526,10 +1674,11 @@ os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] for b in dpkg_keeps_controls : - a='/var/lib/dpkg/info/' + p +'.'+b - if os.path.exists(a): - os.symlink(a,TD+'OLD/CONTROL/'+b) - control_file_pairs.append((b,a)) + z='/var/lib/dpkg/info/' + p +'.'+b + if os.path.exists(z): + os.symlink(z,TD+'OLD/CONTROL/'+b) + control_file_triples.append((b,z,None)) + del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): @@ -1563,8 +1712,6 @@ runtime['patchprogress']=12 - a='' - if DEBUG: a='-v' script_time = - time.time() this_deb_format=DEB_FORMAT @@ -1583,11 +1730,11 @@ elif this_deb_format == 'preunpacked': cmd+=['piped'] + env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, - stdin=open(os.devnull), - stderr=subprocess.PIPE, stdout=temp_name_fd) - progresschar=0.0 - progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh'))) + bufsize=4096,close_fds=True, + stdin=open(os.devnull),env=env, + stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar @@ -1690,13 +1837,19 @@ do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' - for j in F.stderr: - os.write(temp_err_name_fd, j) - progresschar+=len(j) - progress=(int(12.0 + 84.0 * progresschar / progresslen)) - runtime['patchprogress']=progress - if do_progress: - sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) + runtime['patchprogress']=12 + if 'NEW/Size' in params: + NEW_size=int(params['NEW/Size']) + while None == F.poll(): + if os.path.exists(TD+'NEW.file'): + a=os.path.getsize(TD+'NEW.file') + progress=(int(12.0 + 84.0 * a / NEW_size)) + else: + progress=12 + runtime['patchprogress']=progress + time.sleep(0.1) + if do_progress: + sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') @@ -1708,23 +1861,25 @@ runtime['patchprogress']=97 #helper for debugging - def tempos(): + def tempos(f): if os.path.getsize(temp_name): - sys.stderr.write('!! '+temp_name+'\n') + f.append(temp_name) if os.path.getsize(temp_err_name): - sys.stderr.write('!! '+temp_err_name+'\n') + f.append(temp_err_name) - if DEBUG == 0: + if not FORENSIC: def fore(): - sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "-d" ).')+'\n') + return None elif olddeb != '/': def fore(): - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! '+delta+'\n!! '+olddeb+'\n') - tempos() + f=[delta,olddeb] + tempos(f) + return f else: def fore(): temp_fore_name='' + f=[] + tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_") temp_file=os.fdopen(temp_fd,'w') @@ -1732,74 +1887,73 @@ temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') + for i in f: + temp_file.write('PatchLogFile: '+str(i)+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') - forensics_rfc(temp_file,params,control_file_pairs, - linked_file_pairs,diverted,diversions,conf_files) + forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, + diverted,diversions,localepurged,prelink_u_failed) temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! ' + temp_fore_name + '\n') - tempos() - - ##then , really execute the patch + f.append(temp_fore_name) + return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: - fore() - raise DebDeltaError('error in patch.sh.') + f=fore() + raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): - fore() - raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size']) + f=fore() + raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : print ' verifying MD5 for ',os.path.basename(newdeb or delta) m= compute_md5(open(TD+'NEW.file')) if params['NEW/MD5sum'] != m : - fore() - raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) ) + f=fore() + raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta) elif this_deb_format == 'unzipped' : if DO_MD5: m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for control.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) m=compute_md5(subprocess.Popen('ar p "%s" data.tar' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/data.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for data.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for control.tar' ) + raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for data.tar' ) + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) @@ -1867,7 +2021,7 @@ if T : rmtree(T) return r -def do_delta_(olddeb,newdeb,delta,TD): +def do_delta_(olddeb,newdeb,delta,TD,forensic_file=None): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, @@ -1973,7 +2127,7 @@ self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') - self.fd.write('lzma') + self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') self.fd.write('xz') @@ -2559,7 +2713,7 @@ time_corr=0 #################### vvv delta_tar vvv ########################### - def delta_tar(old_filename, new_filename, CWD,\ + def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" @@ -2580,7 +2734,10 @@ oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) - + if old_forensic != None: + #fixme : devices are not supported (but debian policy does not allow them) + old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ + oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname @@ -2605,11 +2762,19 @@ if oldname in skip: if VERBOSE > 2 : print ' skipping ',repr(oldname) + if old_forensic != None: + oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + if old_forensic != None: + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) + oldtar.close() if type(old_filename) == StringType : unlink(TD+old_filename) @@ -2914,6 +3079,13 @@ ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') + if forensic_file==None: + control_forensic=None + data_forensic=None + else: + control_forensic=[] + data_forensic=[] + for name in ar_list_new : newname = 'NEW/'+name system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') @@ -2951,7 +3123,7 @@ if a not in dpkg_keeps_controls: skip.append(a) #delta it - delta_tar(oldname,newname,'CONTROL',skip) + delta_tar(oldname,newname,'CONTROL',control_forensic,skip) script.end_member() elif not NEEDSOLD and name[:8] == 'data.tar' : script.start_member(ar_line, newname, extrachar) @@ -2970,7 +3142,7 @@ def x(): return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.xz | unxz -c') else: assert(0) - delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\ + delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x script.end_member() @@ -3013,6 +3185,9 @@ #script is done script.close() + if forensic_file: + forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) + patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): @@ -3500,8 +3675,25 @@ deltatmp=delta+'_tmp_' ret= None tdir=tempo() + + forensicfile=None + if FORENSICDIR: + if 'Filename' in new: + forensicdirname=delta_dirname(os.path.dirname(new['Filename']),FORENSICDIR) + elif 'File' in new: + forensicdirname=delta_dirname(os.path.dirname(new['File']),FORENSICDIR) + else: + assert(0) + if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act + os.makedirs(forensicdirname) + forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' + a=os.path.join(forensicdirname,forensicbasename) + if not os.path.exists(a): + forensicfile=open(a,'w') + del a + try: - ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir) + ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir, forensic_file=forensicfile) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret except KeyboardInterrupt: if os.path.exists(deltatmp): @@ -3809,7 +4001,8 @@ # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError, s: - if s.errno == 11 : + from errno import EAGAIN + if s.errno == EAGAIN : a=' already locked!' else: a=str(s) @@ -3836,7 +4029,7 @@ patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch - def thread_do_patch(que, no_delta, returns, exitcodes): + def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : print ' Patching thread started. ' debs_size=0 debs_time=0 @@ -3876,6 +4069,7 @@ if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : print ' No deb-policy "e", no download of ',deb_uri + forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return @@ -4170,10 +4364,11 @@ ###################################### end of HTTP stuff ################### start patching thread + forensics=[] patching_thread=threading.Thread( target=thread_do_patch , - args=(patching_queue, no_delta, thread_returns, mainexitcodes) ) + args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() @@ -4453,7 +4648,7 @@ while patching_thread.isAlive(): time.sleep(0.1) - + #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.isAlive(): @@ -4486,7 +4681,9 @@ t=total_time print ' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )} - + + if forensics: + forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff @@ -4535,6 +4732,8 @@ raise SystemExit(5) except DebDeltaError,s: puke('debpatch',s) + if s.logs: + forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception,s: puke('debpatch',s) diff -Nru debdelta-0.44/debdeltas debdelta-0.45/debdeltas --- debdelta-0.44/debdeltas 2011-08-25 11:05:53.000000000 +0000 +++ debdelta-0.45/debdeltas 2011-12-06 16:22:44.000000000 +0000 @@ -169,6 +169,21 @@ DEB_POLICY = ['b','s','e'] DO_PROGRESS = terminalcolumns != None +#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails +#possible values: +# False : do not send them +# True : compute forensic but not send them, just list them +# mail : automatically send by email to default address +# user@domain : automatically send by email to address +# mailto:user@domain : as above +# mutt:user@domain : as above, but use 'mutt', so the user can customize it +# http://domain/cgi : send them automatically thru a CGI script +#Warning: the above is mostly TODO +FORENSIC=False + +#directory tree where forensic info are stored by 'debdeltas' +FORENSICDIR=None + DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') @@ -219,7 +234,7 @@ try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', - 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug', + 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=") ) except getopt.GetoptError,a: sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') @@ -260,6 +275,24 @@ if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) + + elif o == '--forensicdir' : + FORENSICDIR = abspath(expanduser(v)) + if v[-2:] == '//': + FORENSICDIR += '//' + if not os.path.isdir(FORENSICDIR): + sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') + raise SystemExit(3) + + elif o == '--forensic' : + FORENSIC = v + if FORENSIC[:4] == 'http': + try: + import poster + except: + print 'To use the http forensic, you must install the package "python-poster".' + raise SystemExit(3) + elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') @@ -410,7 +443,9 @@ def de_bar(a): if a and a[:2] == './' : a=a[2:] - if a and a[0] == '/' : + elif a == '/.' : + a='' + elif a and a[0] == '/' : a=a[1:] return a @@ -546,7 +581,7 @@ class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. - def __init__(self,s,retriable=False,exitcode=None): + def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) == StringType) self.retriable = retriable if retriable: @@ -559,6 +594,7 @@ else: exitcode = 2 self.exitcode=exitcode + self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') @@ -1116,43 +1152,132 @@ a=a+ ( '%02x' % ord(i) ) return a -def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]): - o.write('Package: '+db['OLD/Package']+'\n') - o.write('Version: '+db['OLD/Version']+'\n') - o.write('Architecture: '+db['OLD/Architecture']+'\n') +def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): + " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" + assert type(diversions) == dict + if type(db) == dict: + for a in sorted(db.keys()): + if a[:3] == 'OLD': + o.write(a[4:]+': '+db[a]+'\n') + else: + for a in sorted(db): + if a[:3] == 'OLD': + o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") - for a in diverted: + for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") - for a in conffiles: + for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") - for a,b in L: - if not os.path.exists(b): - o.write(' NONEXISTENT\n '+b+'\n \n') + for l in sorted(L): + if bytar: + name,mode,tartype,uid,gid,uname,gname,data=l + tmpcopy=None + divert=None + else: + name,divert,tmpcopy=l + if os.path.exists(divert): + fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) + else: + fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' + if tartype == tarfile.REGTYPE: + if tmpcopy and os.path.exists(tmpcopy): + data=hash_to_hex(sha1_hash_file(tmpcopy)) + elif os.path.exists(divert): + data=hash_to_hex(sha1_hash_file(divert)) + if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue - name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b) - if tartype == tarfile.REGTYPE: - data=hash_to_hex(sha1_hash_file(b)) if uname == None: uname=str(uid) if gname == None: gname=str(gid) + name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) - if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles: - o.write(" [conffile]\n") - else: - o.write("\n") - o.write(" "+a+"\n") + if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: + o.write(" [conffile]") + if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: + o.write(" [localpurged]") + if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: + o.write(" [prelink-u failed]") + if divert and not os.path.exists(divert): + o.write(" [missing file %r]" % divert) + if tmpcopy: + o.write(" [prelink-u]") + o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") +def forensic_send(f,forensic=FORENSIC): + " note that f must be a list of lists (or None)" + assert type(f) == list + if not forensic : + if f: + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + return + if not f: + return + if all([(z == None) for z in f]): + print 'Sorry, no forensic logs were generated' + return + if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': + email=EMAIL + if ':' in forensic: + a=forensic.find(':') + email == forensic[a:] + forensic=forensic[:a] + print _("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.") + if forensic in ('mutt','mail'): + raw_input( _('(hit any key)') ) + args=[] + for z in f: + if z: + for j in z: + args+=['-a',j] + subprocess.call(['mutt',email,'-s','delta_failures']+args) + else: + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) + subprocess.call([forensic,'-compose',args]) + return + elif forensic[:4] == 'http': + print _("There were faulty deltas.")+' '+_('Sending logs to server.') + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + #http://atlee.ca/software/poster + import urllib, urllib2, httplib, poster + poster.streaminghttp.register_openers() + datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) + # Create the Request object + request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) + # Actually do the request, and get the response + print ' '+_('Server answers:'),repr(urllib2.urlopen(request).read()) + return + else: + sys.stderr.write(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) + for z in f: + if z: + sys.stderr.write(' '+string.join(z,' ')+'\n') + return + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct @@ -1302,11 +1427,11 @@ #this is not needed in preparing the patch, but may help in forensic conf_files=[] - a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' - if DEBUG and os.path.isfile(a): - #note that filenames have leading / - conf_files=[p for p in open(a).read().split('\n') if p] - del a + z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' + if DEBUG and os.path.isfile(z): + #note that filenames do not have leading / + conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] + del z ### s=patch_check_tmp_space(params,olddeb) @@ -1350,6 +1475,7 @@ elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) + del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 @@ -1399,9 +1525,10 @@ p.close() return s, diverted - localepurged=[] - prelink_u_failed=[] def _symlink_data_tree(pa,TD,diversions,runtime): + localepurged=[] + prelink_u_failed=[] + file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: @@ -1417,8 +1544,8 @@ if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : - a=TD+'OLD/DATA'+orig - d=os.path.dirname(a) + tmpcopy=TD+'OLD/DATA'+orig + d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru @@ -1433,40 +1560,54 @@ prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : - print ' copying/unprelinking ',divert,' to ',a + print ' copying/unprelinking ',divert,' to ', tmpcopy #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 - shutil.copy2(divert,a) - proc=subprocess.Popen(["/usr/sbin/prelink","-u",a],stdin=open(os.devnull),\ + shutil.copy2(divert, tmpcopy) + proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT) out=proc.stdout.read().strip() proc.wait() if proc.returncode: - if not os.path.exists(a): - if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',a,')' - os.symlink(divert, a) + if not os.path.exists(tmpcopy): + if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',tmpcopy,')' + os.symlink(divert, tmpcopy) + prelink_u_failed.append(de_bar(orig)) + unprelink=False elif VERBOSE > 4 : print ' (prelink failed, but file was copied)' - thestat = os.statvfs(a) + thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG: sys.stderr.write('!!'+repr(out)+'\n') elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : sys.stderr.write('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') - prelink_u_failed.append(a) + prelink_u_failed.append(de_bar(orig)) + unprelink=False else: - sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (a,out)) - prelink_u_failed.append(a) + sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (tmpcopy,out)) + prelink_u_failed.append(de_bar(orig)) + unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : print ' symlinking ',divert,' to ',a - os.symlink(divert, a) + os.symlink(divert, tmpcopy) + if unprelink and FORENSIC: + #unfortunately the script will delete the 'tmpcopy', so we hardlink it + z=tempfile.mktemp(prefix=TD) + os.link(tmpcopy,z) + file_triples.append((orig,divert,z)) + else: + file_triples.append((orig,divert,None)) elif not os.path.exists(divert): + file_triples.append((orig,divert,None)) if VERBOSE : print ' Disappeared file? ',divert for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: - localepurged.append(orig) - elif VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig - return s,diverted, prelink_time, prelink_datasize + localepurged.append(de_bar(orig)) + else: + file_triples.append((orig,divert,None)) + if VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig + return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" @@ -1487,8 +1628,14 @@ i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) - control_file_pairs=[] - linked_file_pairs,diverted=[],[] + #initialize, just in case + control_file_triples=[] + file_triples=[] + localepurged=[] + prelink_u_failed=[] + diverted=[] + prelink_time=0 + prelink_datasize=0 ###see into parameters: the patch may need extra info and data @@ -1503,7 +1650,8 @@ elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if olddeb == '/': - linked_file_pairs,diverted,prelink_time,prelink_datasize=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) + file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ + _symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if 'data.tar.bz2' in ar_list_old: @@ -1526,10 +1674,11 @@ os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] for b in dpkg_keeps_controls : - a='/var/lib/dpkg/info/' + p +'.'+b - if os.path.exists(a): - os.symlink(a,TD+'OLD/CONTROL/'+b) - control_file_pairs.append((b,a)) + z='/var/lib/dpkg/info/' + p +'.'+b + if os.path.exists(z): + os.symlink(z,TD+'OLD/CONTROL/'+b) + control_file_triples.append((b,z,None)) + del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): @@ -1563,8 +1712,6 @@ runtime['patchprogress']=12 - a='' - if DEBUG: a='-v' script_time = - time.time() this_deb_format=DEB_FORMAT @@ -1583,11 +1730,11 @@ elif this_deb_format == 'preunpacked': cmd+=['piped'] + env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, - stdin=open(os.devnull), - stderr=subprocess.PIPE, stdout=temp_name_fd) - progresschar=0.0 - progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh'))) + bufsize=4096,close_fds=True, + stdin=open(os.devnull),env=env, + stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar @@ -1690,13 +1837,19 @@ do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' - for j in F.stderr: - os.write(temp_err_name_fd, j) - progresschar+=len(j) - progress=(int(12.0 + 84.0 * progresschar / progresslen)) - runtime['patchprogress']=progress - if do_progress: - sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) + runtime['patchprogress']=12 + if 'NEW/Size' in params: + NEW_size=int(params['NEW/Size']) + while None == F.poll(): + if os.path.exists(TD+'NEW.file'): + a=os.path.getsize(TD+'NEW.file') + progress=(int(12.0 + 84.0 * a / NEW_size)) + else: + progress=12 + runtime['patchprogress']=progress + time.sleep(0.1) + if do_progress: + sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') @@ -1708,23 +1861,25 @@ runtime['patchprogress']=97 #helper for debugging - def tempos(): + def tempos(f): if os.path.getsize(temp_name): - sys.stderr.write('!! '+temp_name+'\n') + f.append(temp_name) if os.path.getsize(temp_err_name): - sys.stderr.write('!! '+temp_err_name+'\n') + f.append(temp_err_name) - if DEBUG == 0: + if not FORENSIC: def fore(): - sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "-d" ).')+'\n') + return None elif olddeb != '/': def fore(): - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! '+delta+'\n!! '+olddeb+'\n') - tempos() + f=[delta,olddeb] + tempos(f) + return f else: def fore(): temp_fore_name='' + f=[] + tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_") temp_file=os.fdopen(temp_fd,'w') @@ -1732,74 +1887,73 @@ temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') + for i in f: + temp_file.write('PatchLogFile: '+str(i)+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') - forensics_rfc(temp_file,params,control_file_pairs, - linked_file_pairs,diverted,diversions,conf_files) + forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, + diverted,diversions,localepurged,prelink_u_failed) temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! ' + temp_fore_name + '\n') - tempos() - - ##then , really execute the patch + f.append(temp_fore_name) + return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: - fore() - raise DebDeltaError('error in patch.sh.') + f=fore() + raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): - fore() - raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size']) + f=fore() + raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : print ' verifying MD5 for ',os.path.basename(newdeb or delta) m= compute_md5(open(TD+'NEW.file')) if params['NEW/MD5sum'] != m : - fore() - raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) ) + f=fore() + raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta) elif this_deb_format == 'unzipped' : if DO_MD5: m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for control.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) m=compute_md5(subprocess.Popen('ar p "%s" data.tar' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/data.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for data.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for control.tar' ) + raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for data.tar' ) + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) @@ -1867,7 +2021,7 @@ if T : rmtree(T) return r -def do_delta_(olddeb,newdeb,delta,TD): +def do_delta_(olddeb,newdeb,delta,TD,forensic_file=None): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, @@ -1973,7 +2127,7 @@ self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') - self.fd.write('lzma') + self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') self.fd.write('xz') @@ -2559,7 +2713,7 @@ time_corr=0 #################### vvv delta_tar vvv ########################### - def delta_tar(old_filename, new_filename, CWD,\ + def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" @@ -2580,7 +2734,10 @@ oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) - + if old_forensic != None: + #fixme : devices are not supported (but debian policy does not allow them) + old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ + oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname @@ -2605,11 +2762,19 @@ if oldname in skip: if VERBOSE > 2 : print ' skipping ',repr(oldname) + if old_forensic != None: + oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + if old_forensic != None: + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) + oldtar.close() if type(old_filename) == StringType : unlink(TD+old_filename) @@ -2914,6 +3079,13 @@ ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') + if forensic_file==None: + control_forensic=None + data_forensic=None + else: + control_forensic=[] + data_forensic=[] + for name in ar_list_new : newname = 'NEW/'+name system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') @@ -2951,7 +3123,7 @@ if a not in dpkg_keeps_controls: skip.append(a) #delta it - delta_tar(oldname,newname,'CONTROL',skip) + delta_tar(oldname,newname,'CONTROL',control_forensic,skip) script.end_member() elif not NEEDSOLD and name[:8] == 'data.tar' : script.start_member(ar_line, newname, extrachar) @@ -2970,7 +3142,7 @@ def x(): return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.xz | unxz -c') else: assert(0) - delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\ + delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x script.end_member() @@ -3013,6 +3185,9 @@ #script is done script.close() + if forensic_file: + forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) + patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): @@ -3500,8 +3675,25 @@ deltatmp=delta+'_tmp_' ret= None tdir=tempo() + + forensicfile=None + if FORENSICDIR: + if 'Filename' in new: + forensicdirname=delta_dirname(os.path.dirname(new['Filename']),FORENSICDIR) + elif 'File' in new: + forensicdirname=delta_dirname(os.path.dirname(new['File']),FORENSICDIR) + else: + assert(0) + if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act + os.makedirs(forensicdirname) + forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' + a=os.path.join(forensicdirname,forensicbasename) + if not os.path.exists(a): + forensicfile=open(a,'w') + del a + try: - ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir) + ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir, forensic_file=forensicfile) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret except KeyboardInterrupt: if os.path.exists(deltatmp): @@ -3809,7 +4001,8 @@ # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError, s: - if s.errno == 11 : + from errno import EAGAIN + if s.errno == EAGAIN : a=' already locked!' else: a=str(s) @@ -3836,7 +4029,7 @@ patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch - def thread_do_patch(que, no_delta, returns, exitcodes): + def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : print ' Patching thread started. ' debs_size=0 debs_time=0 @@ -3876,6 +4069,7 @@ if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : print ' No deb-policy "e", no download of ',deb_uri + forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return @@ -4170,10 +4364,11 @@ ###################################### end of HTTP stuff ################### start patching thread + forensics=[] patching_thread=threading.Thread( target=thread_do_patch , - args=(patching_queue, no_delta, thread_returns, mainexitcodes) ) + args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() @@ -4453,7 +4648,7 @@ while patching_thread.isAlive(): time.sleep(0.1) - + #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.isAlive(): @@ -4486,7 +4681,9 @@ t=total_time print ' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )} - + + if forensics: + forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff @@ -4535,6 +4732,8 @@ raise SystemExit(5) except DebDeltaError,s: puke('debpatch',s) + if s.logs: + forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception,s: puke('debpatch',s) diff -Nru debdelta-0.44/debdeltas.1 debdelta-0.45/debdeltas.1 --- debdelta-0.44/debdeltas.1 2011-03-31 09:32:26.000000000 +0000 +++ debdelta-0.45/debdeltas.1 2011-12-06 13:12:01.000000000 +0000 @@ -85,6 +85,11 @@ index, it is not an error if files do not exist, as long as they have been moved in a --alt directory. Note that, if no --old is specified, then no deltas will be generated. +.TP +\fB\-\-forensicdir \fIDIR +write hashes files; these are to be compared with those produced by +.I debdelta-upgrade --forensic=... +when a delta fails .SH The double slash If a directory path is provided as argument to --dir, and it ends in // , diff -Nru debdelta-0.44/debdelta-upgrade debdelta-0.45/debdelta-upgrade --- debdelta-0.44/debdelta-upgrade 2011-08-25 11:05:53.000000000 +0000 +++ debdelta-0.45/debdelta-upgrade 2011-12-06 16:22:44.000000000 +0000 @@ -169,6 +169,21 @@ DEB_POLICY = ['b','s','e'] DO_PROGRESS = terminalcolumns != None +#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails +#possible values: +# False : do not send them +# True : compute forensic but not send them, just list them +# mail : automatically send by email to default address +# user@domain : automatically send by email to address +# mailto:user@domain : as above +# mutt:user@domain : as above, but use 'mutt', so the user can customize it +# http://domain/cgi : send them automatically thru a CGI script +#Warning: the above is mostly TODO +FORENSIC=False + +#directory tree where forensic info are stored by 'debdeltas' +FORENSICDIR=None + DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') @@ -219,7 +234,7 @@ try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', - 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug', + 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=") ) except getopt.GetoptError,a: sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') @@ -260,6 +275,24 @@ if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) + + elif o == '--forensicdir' : + FORENSICDIR = abspath(expanduser(v)) + if v[-2:] == '//': + FORENSICDIR += '//' + if not os.path.isdir(FORENSICDIR): + sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') + raise SystemExit(3) + + elif o == '--forensic' : + FORENSIC = v + if FORENSIC[:4] == 'http': + try: + import poster + except: + print 'To use the http forensic, you must install the package "python-poster".' + raise SystemExit(3) + elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') @@ -410,7 +443,9 @@ def de_bar(a): if a and a[:2] == './' : a=a[2:] - if a and a[0] == '/' : + elif a == '/.' : + a='' + elif a and a[0] == '/' : a=a[1:] return a @@ -546,7 +581,7 @@ class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. - def __init__(self,s,retriable=False,exitcode=None): + def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) == StringType) self.retriable = retriable if retriable: @@ -559,6 +594,7 @@ else: exitcode = 2 self.exitcode=exitcode + self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') @@ -1116,43 +1152,132 @@ a=a+ ( '%02x' % ord(i) ) return a -def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]): - o.write('Package: '+db['OLD/Package']+'\n') - o.write('Version: '+db['OLD/Version']+'\n') - o.write('Architecture: '+db['OLD/Architecture']+'\n') +def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): + " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" + assert type(diversions) == dict + if type(db) == dict: + for a in sorted(db.keys()): + if a[:3] == 'OLD': + o.write(a[4:]+': '+db[a]+'\n') + else: + for a in sorted(db): + if a[:3] == 'OLD': + o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") - for a in diverted: + for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") - for a in conffiles: + for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") - for a,b in L: - if not os.path.exists(b): - o.write(' NONEXISTENT\n '+b+'\n \n') + for l in sorted(L): + if bytar: + name,mode,tartype,uid,gid,uname,gname,data=l + tmpcopy=None + divert=None + else: + name,divert,tmpcopy=l + if os.path.exists(divert): + fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) + else: + fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' + if tartype == tarfile.REGTYPE: + if tmpcopy and os.path.exists(tmpcopy): + data=hash_to_hex(sha1_hash_file(tmpcopy)) + elif os.path.exists(divert): + data=hash_to_hex(sha1_hash_file(divert)) + if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue - name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b) - if tartype == tarfile.REGTYPE: - data=hash_to_hex(sha1_hash_file(b)) if uname == None: uname=str(uid) if gname == None: gname=str(gid) + name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) - if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles: - o.write(" [conffile]\n") - else: - o.write("\n") - o.write(" "+a+"\n") + if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: + o.write(" [conffile]") + if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: + o.write(" [localpurged]") + if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: + o.write(" [prelink-u failed]") + if divert and not os.path.exists(divert): + o.write(" [missing file %r]" % divert) + if tmpcopy: + o.write(" [prelink-u]") + o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") +def forensic_send(f,forensic=FORENSIC): + " note that f must be a list of lists (or None)" + assert type(f) == list + if not forensic : + if f: + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + return + if not f: + return + if all([(z == None) for z in f]): + print 'Sorry, no forensic logs were generated' + return + if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': + email=EMAIL + if ':' in forensic: + a=forensic.find(':') + email == forensic[a:] + forensic=forensic[:a] + print _("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.") + if forensic in ('mutt','mail'): + raw_input( _('(hit any key)') ) + args=[] + for z in f: + if z: + for j in z: + args+=['-a',j] + subprocess.call(['mutt',email,'-s','delta_failures']+args) + else: + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) + subprocess.call([forensic,'-compose',args]) + return + elif forensic[:4] == 'http': + print _("There were faulty deltas.")+' '+_('Sending logs to server.') + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + #http://atlee.ca/software/poster + import urllib, urllib2, httplib, poster + poster.streaminghttp.register_openers() + datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) + # Create the Request object + request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) + # Actually do the request, and get the response + print ' '+_('Server answers:'),repr(urllib2.urlopen(request).read()) + return + else: + sys.stderr.write(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) + for z in f: + if z: + sys.stderr.write(' '+string.join(z,' ')+'\n') + return + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct @@ -1302,11 +1427,11 @@ #this is not needed in preparing the patch, but may help in forensic conf_files=[] - a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' - if DEBUG and os.path.isfile(a): - #note that filenames have leading / - conf_files=[p for p in open(a).read().split('\n') if p] - del a + z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' + if DEBUG and os.path.isfile(z): + #note that filenames do not have leading / + conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] + del z ### s=patch_check_tmp_space(params,olddeb) @@ -1350,6 +1475,7 @@ elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) + del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 @@ -1399,9 +1525,10 @@ p.close() return s, diverted - localepurged=[] - prelink_u_failed=[] def _symlink_data_tree(pa,TD,diversions,runtime): + localepurged=[] + prelink_u_failed=[] + file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: @@ -1417,8 +1544,8 @@ if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : - a=TD+'OLD/DATA'+orig - d=os.path.dirname(a) + tmpcopy=TD+'OLD/DATA'+orig + d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru @@ -1433,40 +1560,54 @@ prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : - print ' copying/unprelinking ',divert,' to ',a + print ' copying/unprelinking ',divert,' to ', tmpcopy #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 - shutil.copy2(divert,a) - proc=subprocess.Popen(["/usr/sbin/prelink","-u",a],stdin=open(os.devnull),\ + shutil.copy2(divert, tmpcopy) + proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT) out=proc.stdout.read().strip() proc.wait() if proc.returncode: - if not os.path.exists(a): - if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',a,')' - os.symlink(divert, a) + if not os.path.exists(tmpcopy): + if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',tmpcopy,')' + os.symlink(divert, tmpcopy) + prelink_u_failed.append(de_bar(orig)) + unprelink=False elif VERBOSE > 4 : print ' (prelink failed, but file was copied)' - thestat = os.statvfs(a) + thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG: sys.stderr.write('!!'+repr(out)+'\n') elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : sys.stderr.write('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') - prelink_u_failed.append(a) + prelink_u_failed.append(de_bar(orig)) + unprelink=False else: - sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (a,out)) - prelink_u_failed.append(a) + sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (tmpcopy,out)) + prelink_u_failed.append(de_bar(orig)) + unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : print ' symlinking ',divert,' to ',a - os.symlink(divert, a) + os.symlink(divert, tmpcopy) + if unprelink and FORENSIC: + #unfortunately the script will delete the 'tmpcopy', so we hardlink it + z=tempfile.mktemp(prefix=TD) + os.link(tmpcopy,z) + file_triples.append((orig,divert,z)) + else: + file_triples.append((orig,divert,None)) elif not os.path.exists(divert): + file_triples.append((orig,divert,None)) if VERBOSE : print ' Disappeared file? ',divert for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: - localepurged.append(orig) - elif VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig - return s,diverted, prelink_time, prelink_datasize + localepurged.append(de_bar(orig)) + else: + file_triples.append((orig,divert,None)) + if VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig + return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" @@ -1487,8 +1628,14 @@ i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) - control_file_pairs=[] - linked_file_pairs,diverted=[],[] + #initialize, just in case + control_file_triples=[] + file_triples=[] + localepurged=[] + prelink_u_failed=[] + diverted=[] + prelink_time=0 + prelink_datasize=0 ###see into parameters: the patch may need extra info and data @@ -1503,7 +1650,8 @@ elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if olddeb == '/': - linked_file_pairs,diverted,prelink_time,prelink_datasize=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) + file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ + _symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if 'data.tar.bz2' in ar_list_old: @@ -1526,10 +1674,11 @@ os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] for b in dpkg_keeps_controls : - a='/var/lib/dpkg/info/' + p +'.'+b - if os.path.exists(a): - os.symlink(a,TD+'OLD/CONTROL/'+b) - control_file_pairs.append((b,a)) + z='/var/lib/dpkg/info/' + p +'.'+b + if os.path.exists(z): + os.symlink(z,TD+'OLD/CONTROL/'+b) + control_file_triples.append((b,z,None)) + del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): @@ -1563,8 +1712,6 @@ runtime['patchprogress']=12 - a='' - if DEBUG: a='-v' script_time = - time.time() this_deb_format=DEB_FORMAT @@ -1583,11 +1730,11 @@ elif this_deb_format == 'preunpacked': cmd+=['piped'] + env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, - stdin=open(os.devnull), - stderr=subprocess.PIPE, stdout=temp_name_fd) - progresschar=0.0 - progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh'))) + bufsize=4096,close_fds=True, + stdin=open(os.devnull),env=env, + stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar @@ -1690,13 +1837,19 @@ do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' - for j in F.stderr: - os.write(temp_err_name_fd, j) - progresschar+=len(j) - progress=(int(12.0 + 84.0 * progresschar / progresslen)) - runtime['patchprogress']=progress - if do_progress: - sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) + runtime['patchprogress']=12 + if 'NEW/Size' in params: + NEW_size=int(params['NEW/Size']) + while None == F.poll(): + if os.path.exists(TD+'NEW.file'): + a=os.path.getsize(TD+'NEW.file') + progress=(int(12.0 + 84.0 * a / NEW_size)) + else: + progress=12 + runtime['patchprogress']=progress + time.sleep(0.1) + if do_progress: + sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') @@ -1708,23 +1861,25 @@ runtime['patchprogress']=97 #helper for debugging - def tempos(): + def tempos(f): if os.path.getsize(temp_name): - sys.stderr.write('!! '+temp_name+'\n') + f.append(temp_name) if os.path.getsize(temp_err_name): - sys.stderr.write('!! '+temp_err_name+'\n') + f.append(temp_err_name) - if DEBUG == 0: + if not FORENSIC: def fore(): - sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "-d" ).')+'\n') + return None elif olddeb != '/': def fore(): - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! '+delta+'\n!! '+olddeb+'\n') - tempos() + f=[delta,olddeb] + tempos(f) + return f else: def fore(): temp_fore_name='' + f=[] + tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_") temp_file=os.fdopen(temp_fd,'w') @@ -1732,74 +1887,73 @@ temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') + for i in f: + temp_file.write('PatchLogFile: '+str(i)+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') - forensics_rfc(temp_file,params,control_file_pairs, - linked_file_pairs,diverted,diversions,conf_files) + forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, + diverted,diversions,localepurged,prelink_u_failed) temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! ' + temp_fore_name + '\n') - tempos() - - ##then , really execute the patch + f.append(temp_fore_name) + return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: - fore() - raise DebDeltaError('error in patch.sh.') + f=fore() + raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): - fore() - raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size']) + f=fore() + raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : print ' verifying MD5 for ',os.path.basename(newdeb or delta) m= compute_md5(open(TD+'NEW.file')) if params['NEW/MD5sum'] != m : - fore() - raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) ) + f=fore() + raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta) elif this_deb_format == 'unzipped' : if DO_MD5: m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for control.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) m=compute_md5(subprocess.Popen('ar p "%s" data.tar' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/data.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for data.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for control.tar' ) + raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for data.tar' ) + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) @@ -1867,7 +2021,7 @@ if T : rmtree(T) return r -def do_delta_(olddeb,newdeb,delta,TD): +def do_delta_(olddeb,newdeb,delta,TD,forensic_file=None): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, @@ -1973,7 +2127,7 @@ self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') - self.fd.write('lzma') + self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') self.fd.write('xz') @@ -2559,7 +2713,7 @@ time_corr=0 #################### vvv delta_tar vvv ########################### - def delta_tar(old_filename, new_filename, CWD,\ + def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" @@ -2580,7 +2734,10 @@ oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) - + if old_forensic != None: + #fixme : devices are not supported (but debian policy does not allow them) + old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ + oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname @@ -2605,11 +2762,19 @@ if oldname in skip: if VERBOSE > 2 : print ' skipping ',repr(oldname) + if old_forensic != None: + oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + if old_forensic != None: + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) + oldtar.close() if type(old_filename) == StringType : unlink(TD+old_filename) @@ -2914,6 +3079,13 @@ ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') + if forensic_file==None: + control_forensic=None + data_forensic=None + else: + control_forensic=[] + data_forensic=[] + for name in ar_list_new : newname = 'NEW/'+name system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') @@ -2951,7 +3123,7 @@ if a not in dpkg_keeps_controls: skip.append(a) #delta it - delta_tar(oldname,newname,'CONTROL',skip) + delta_tar(oldname,newname,'CONTROL',control_forensic,skip) script.end_member() elif not NEEDSOLD and name[:8] == 'data.tar' : script.start_member(ar_line, newname, extrachar) @@ -2970,7 +3142,7 @@ def x(): return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.xz | unxz -c') else: assert(0) - delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\ + delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x script.end_member() @@ -3013,6 +3185,9 @@ #script is done script.close() + if forensic_file: + forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) + patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): @@ -3500,8 +3675,25 @@ deltatmp=delta+'_tmp_' ret= None tdir=tempo() + + forensicfile=None + if FORENSICDIR: + if 'Filename' in new: + forensicdirname=delta_dirname(os.path.dirname(new['Filename']),FORENSICDIR) + elif 'File' in new: + forensicdirname=delta_dirname(os.path.dirname(new['File']),FORENSICDIR) + else: + assert(0) + if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act + os.makedirs(forensicdirname) + forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' + a=os.path.join(forensicdirname,forensicbasename) + if not os.path.exists(a): + forensicfile=open(a,'w') + del a + try: - ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir) + ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir, forensic_file=forensicfile) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret except KeyboardInterrupt: if os.path.exists(deltatmp): @@ -3809,7 +4001,8 @@ # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError, s: - if s.errno == 11 : + from errno import EAGAIN + if s.errno == EAGAIN : a=' already locked!' else: a=str(s) @@ -3836,7 +4029,7 @@ patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch - def thread_do_patch(que, no_delta, returns, exitcodes): + def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : print ' Patching thread started. ' debs_size=0 debs_time=0 @@ -3876,6 +4069,7 @@ if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : print ' No deb-policy "e", no download of ',deb_uri + forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return @@ -4170,10 +4364,11 @@ ###################################### end of HTTP stuff ################### start patching thread + forensics=[] patching_thread=threading.Thread( target=thread_do_patch , - args=(patching_queue, no_delta, thread_returns, mainexitcodes) ) + args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() @@ -4453,7 +4648,7 @@ while patching_thread.isAlive(): time.sleep(0.1) - + #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.isAlive(): @@ -4486,7 +4681,9 @@ t=total_time print ' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )} - + + if forensics: + forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff @@ -4535,6 +4732,8 @@ raise SystemExit(5) except DebDeltaError,s: puke('debpatch',s) + if s.logs: + forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception,s: puke('debpatch',s) diff -Nru debdelta-0.44/debdelta-upgrade.1 debdelta-0.45/debdelta-upgrade.1 --- debdelta-0.44/debdelta-upgrade.1 2011-05-03 07:28:00.000000000 +0000 +++ debdelta-0.45/debdelta-upgrade.1 2011-12-06 16:28:38.000000000 +0000 @@ -52,10 +52,10 @@ \fB\-k keep temporary files (use for debugging). .TP -.B \-A \--accept-unsigned +\fB \-A \--accept-unsigned accept unsigned deltas. .TP -.BI \--gpg-home +\fB \-\-gpg-home specify a different home for GnuPG, default for root is .I /etc/debdelta/gnupg @@ -64,6 +64,18 @@ in .BR gpg(1) for details. +.TP +\fB \-\-forensic \fIMETHOD +if a delta fails, report logs so that the problem may be addressed. +Method may be + do + just prepare logs and say where they are + mutt + send logs by email using mutt + icedove + send logs by email using icedove (as root!) + http + send by http (the easiest and most recommended method!) .SH EXAMPLES diff -Nru debdelta-0.44/debian/changelog debdelta-0.45/debian/changelog --- debdelta-0.44/debian/changelog 2011-08-28 08:17:56.000000000 +0000 +++ debdelta-0.45/debian/changelog 2011-12-06 16:27:18.000000000 +0000 @@ -1,3 +1,16 @@ +debdelta (0.45) unstable; urgency=low + + * debdelta-upgrade/debpatch : new option --forensic, + to report a log when a delta fails + * debdelta/debdeltas : new option --forensicdir, + to store hashes to check above reports + * do not use hardcoded errno values, + thanks to Pino Toscano (Closes: #640627). + * typo in Recommends: xz -> xz-utils, + thanks to Eugene V. Lyubimkin (Closes: #641189). + + -- A Mennucc1 Tue, 06 Dec 2011 17:27:16 +0100 + debdelta (0.44) unstable; urgency=low * support xz compression for data.tar part in .deb diff -Nru debdelta-0.44/debian/control debdelta-0.45/debian/control --- debdelta-0.44/debian/control 2011-08-27 10:11:21.000000000 +0000 +++ debdelta-0.45/debian/control 2011-12-06 12:59:12.000000000 +0000 @@ -10,7 +10,7 @@ Package: debdelta Architecture: any Depends: python, bzip2, binutils, ${shlibs:Depends} -Recommends: python-apt, xdelta3, xdelta, lzma, xz, xdelta, bsdiff, gnupg2, gnupg-agent +Recommends: python-apt, xdelta3, xdelta, lzma, xz-utils, xdelta, bsdiff, gnupg2, gnupg-agent Conflicts: xdelta3 (<< 0y.dfsg-1) Enhances: cupt Suggests: debdelta-doc diff -Nru debdelta-0.44/debpatch debdelta-0.45/debpatch --- debdelta-0.44/debpatch 2011-08-25 11:05:53.000000000 +0000 +++ debdelta-0.45/debpatch 2011-12-06 16:22:44.000000000 +0000 @@ -169,6 +169,21 @@ DEB_POLICY = ['b','s','e'] DO_PROGRESS = terminalcolumns != None +#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails +#possible values: +# False : do not send them +# True : compute forensic but not send them, just list them +# mail : automatically send by email to default address +# user@domain : automatically send by email to address +# mailto:user@domain : as above +# mutt:user@domain : as above, but use 'mutt', so the user can customize it +# http://domain/cgi : send them automatically thru a CGI script +#Warning: the above is mostly TODO +FORENSIC=False + +#directory tree where forensic info are stored by 'debdeltas' +FORENSICDIR=None + DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') @@ -219,7 +234,7 @@ try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', - 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug', + 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=") ) except getopt.GetoptError,a: sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') @@ -260,6 +275,24 @@ if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) + + elif o == '--forensicdir' : + FORENSICDIR = abspath(expanduser(v)) + if v[-2:] == '//': + FORENSICDIR += '//' + if not os.path.isdir(FORENSICDIR): + sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') + raise SystemExit(3) + + elif o == '--forensic' : + FORENSIC = v + if FORENSIC[:4] == 'http': + try: + import poster + except: + print 'To use the http forensic, you must install the package "python-poster".' + raise SystemExit(3) + elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') @@ -410,7 +443,9 @@ def de_bar(a): if a and a[:2] == './' : a=a[2:] - if a and a[0] == '/' : + elif a == '/.' : + a='' + elif a and a[0] == '/' : a=a[1:] return a @@ -546,7 +581,7 @@ class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. - def __init__(self,s,retriable=False,exitcode=None): + def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) == StringType) self.retriable = retriable if retriable: @@ -559,6 +594,7 @@ else: exitcode = 2 self.exitcode=exitcode + self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') @@ -1116,43 +1152,132 @@ a=a+ ( '%02x' % ord(i) ) return a -def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]): - o.write('Package: '+db['OLD/Package']+'\n') - o.write('Version: '+db['OLD/Version']+'\n') - o.write('Architecture: '+db['OLD/Architecture']+'\n') +def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): + " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" + assert type(diversions) == dict + if type(db) == dict: + for a in sorted(db.keys()): + if a[:3] == 'OLD': + o.write(a[4:]+': '+db[a]+'\n') + else: + for a in sorted(db): + if a[:3] == 'OLD': + o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") - for a in diverted: + for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") - for a in conffiles: + for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") - for a,b in L: - if not os.path.exists(b): - o.write(' NONEXISTENT\n '+b+'\n \n') + for l in sorted(L): + if bytar: + name,mode,tartype,uid,gid,uname,gname,data=l + tmpcopy=None + divert=None + else: + name,divert,tmpcopy=l + if os.path.exists(divert): + fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) + else: + fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' + if tartype == tarfile.REGTYPE: + if tmpcopy and os.path.exists(tmpcopy): + data=hash_to_hex(sha1_hash_file(tmpcopy)) + elif os.path.exists(divert): + data=hash_to_hex(sha1_hash_file(divert)) + if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue - name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b) - if tartype == tarfile.REGTYPE: - data=hash_to_hex(sha1_hash_file(b)) if uname == None: uname=str(uid) if gname == None: gname=str(gid) + name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) - if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles: - o.write(" [conffile]\n") - else: - o.write("\n") - o.write(" "+a+"\n") + if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: + o.write(" [conffile]") + if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: + o.write(" [localpurged]") + if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: + o.write(" [prelink-u failed]") + if divert and not os.path.exists(divert): + o.write(" [missing file %r]" % divert) + if tmpcopy: + o.write(" [prelink-u]") + o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") +def forensic_send(f,forensic=FORENSIC): + " note that f must be a list of lists (or None)" + assert type(f) == list + if not forensic : + if f: + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + return + if not f: + return + if all([(z == None) for z in f]): + print 'Sorry, no forensic logs were generated' + return + if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': + email=EMAIL + if ':' in forensic: + a=forensic.find(':') + email == forensic[a:] + forensic=forensic[:a] + print _("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.") + if forensic in ('mutt','mail'): + raw_input( _('(hit any key)') ) + args=[] + for z in f: + if z: + for j in z: + args+=['-a',j] + subprocess.call(['mutt',email,'-s','delta_failures']+args) + else: + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) + subprocess.call([forensic,'-compose',args]) + return + elif forensic[:4] == 'http': + print _("There were faulty deltas.")+' '+_('Sending logs to server.') + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + #http://atlee.ca/software/poster + import urllib, urllib2, httplib, poster + poster.streaminghttp.register_openers() + datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) + # Create the Request object + request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) + # Actually do the request, and get the response + print ' '+_('Server answers:'),repr(urllib2.urlopen(request).read()) + return + else: + sys.stderr.write(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) + for z in f: + if z: + sys.stderr.write(' '+string.join(z,' ')+'\n') + return + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct @@ -1302,11 +1427,11 @@ #this is not needed in preparing the patch, but may help in forensic conf_files=[] - a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' - if DEBUG and os.path.isfile(a): - #note that filenames have leading / - conf_files=[p for p in open(a).read().split('\n') if p] - del a + z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' + if DEBUG and os.path.isfile(z): + #note that filenames do not have leading / + conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] + del z ### s=patch_check_tmp_space(params,olddeb) @@ -1350,6 +1475,7 @@ elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) + del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 @@ -1399,9 +1525,10 @@ p.close() return s, diverted - localepurged=[] - prelink_u_failed=[] def _symlink_data_tree(pa,TD,diversions,runtime): + localepurged=[] + prelink_u_failed=[] + file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: @@ -1417,8 +1544,8 @@ if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : - a=TD+'OLD/DATA'+orig - d=os.path.dirname(a) + tmpcopy=TD+'OLD/DATA'+orig + d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru @@ -1433,40 +1560,54 @@ prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : - print ' copying/unprelinking ',divert,' to ',a + print ' copying/unprelinking ',divert,' to ', tmpcopy #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 - shutil.copy2(divert,a) - proc=subprocess.Popen(["/usr/sbin/prelink","-u",a],stdin=open(os.devnull),\ + shutil.copy2(divert, tmpcopy) + proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT) out=proc.stdout.read().strip() proc.wait() if proc.returncode: - if not os.path.exists(a): - if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',a,')' - os.symlink(divert, a) + if not os.path.exists(tmpcopy): + if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',tmpcopy,')' + os.symlink(divert, tmpcopy) + prelink_u_failed.append(de_bar(orig)) + unprelink=False elif VERBOSE > 4 : print ' (prelink failed, but file was copied)' - thestat = os.statvfs(a) + thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG: sys.stderr.write('!!'+repr(out)+'\n') elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : sys.stderr.write('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') - prelink_u_failed.append(a) + prelink_u_failed.append(de_bar(orig)) + unprelink=False else: - sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (a,out)) - prelink_u_failed.append(a) + sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (tmpcopy,out)) + prelink_u_failed.append(de_bar(orig)) + unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : print ' symlinking ',divert,' to ',a - os.symlink(divert, a) + os.symlink(divert, tmpcopy) + if unprelink and FORENSIC: + #unfortunately the script will delete the 'tmpcopy', so we hardlink it + z=tempfile.mktemp(prefix=TD) + os.link(tmpcopy,z) + file_triples.append((orig,divert,z)) + else: + file_triples.append((orig,divert,None)) elif not os.path.exists(divert): + file_triples.append((orig,divert,None)) if VERBOSE : print ' Disappeared file? ',divert for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: - localepurged.append(orig) - elif VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig - return s,diverted, prelink_time, prelink_datasize + localepurged.append(de_bar(orig)) + else: + file_triples.append((orig,divert,None)) + if VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig + return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" @@ -1487,8 +1628,14 @@ i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) - control_file_pairs=[] - linked_file_pairs,diverted=[],[] + #initialize, just in case + control_file_triples=[] + file_triples=[] + localepurged=[] + prelink_u_failed=[] + diverted=[] + prelink_time=0 + prelink_datasize=0 ###see into parameters: the patch may need extra info and data @@ -1503,7 +1650,8 @@ elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if olddeb == '/': - linked_file_pairs,diverted,prelink_time,prelink_datasize=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) + file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ + _symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if 'data.tar.bz2' in ar_list_old: @@ -1526,10 +1674,11 @@ os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] for b in dpkg_keeps_controls : - a='/var/lib/dpkg/info/' + p +'.'+b - if os.path.exists(a): - os.symlink(a,TD+'OLD/CONTROL/'+b) - control_file_pairs.append((b,a)) + z='/var/lib/dpkg/info/' + p +'.'+b + if os.path.exists(z): + os.symlink(z,TD+'OLD/CONTROL/'+b) + control_file_triples.append((b,z,None)) + del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): @@ -1563,8 +1712,6 @@ runtime['patchprogress']=12 - a='' - if DEBUG: a='-v' script_time = - time.time() this_deb_format=DEB_FORMAT @@ -1583,11 +1730,11 @@ elif this_deb_format == 'preunpacked': cmd+=['piped'] + env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, - stdin=open(os.devnull), - stderr=subprocess.PIPE, stdout=temp_name_fd) - progresschar=0.0 - progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh'))) + bufsize=4096,close_fds=True, + stdin=open(os.devnull),env=env, + stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar @@ -1690,13 +1837,19 @@ do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' - for j in F.stderr: - os.write(temp_err_name_fd, j) - progresschar+=len(j) - progress=(int(12.0 + 84.0 * progresschar / progresslen)) - runtime['patchprogress']=progress - if do_progress: - sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) + runtime['patchprogress']=12 + if 'NEW/Size' in params: + NEW_size=int(params['NEW/Size']) + while None == F.poll(): + if os.path.exists(TD+'NEW.file'): + a=os.path.getsize(TD+'NEW.file') + progress=(int(12.0 + 84.0 * a / NEW_size)) + else: + progress=12 + runtime['patchprogress']=progress + time.sleep(0.1) + if do_progress: + sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') @@ -1708,23 +1861,25 @@ runtime['patchprogress']=97 #helper for debugging - def tempos(): + def tempos(f): if os.path.getsize(temp_name): - sys.stderr.write('!! '+temp_name+'\n') + f.append(temp_name) if os.path.getsize(temp_err_name): - sys.stderr.write('!! '+temp_err_name+'\n') + f.append(temp_err_name) - if DEBUG == 0: + if not FORENSIC: def fore(): - sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "-d" ).')+'\n') + return None elif olddeb != '/': def fore(): - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! '+delta+'\n!! '+olddeb+'\n') - tempos() + f=[delta,olddeb] + tempos(f) + return f else: def fore(): temp_fore_name='' + f=[] + tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_") temp_file=os.fdopen(temp_fd,'w') @@ -1732,74 +1887,73 @@ temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') + for i in f: + temp_file.write('PatchLogFile: '+str(i)+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') - forensics_rfc(temp_file,params,control_file_pairs, - linked_file_pairs,diverted,diversions,conf_files) + forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, + diverted,diversions,localepurged,prelink_u_failed) temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! ' + temp_fore_name + '\n') - tempos() - - ##then , really execute the patch + f.append(temp_fore_name) + return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: - fore() - raise DebDeltaError('error in patch.sh.') + f=fore() + raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): - fore() - raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size']) + f=fore() + raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : print ' verifying MD5 for ',os.path.basename(newdeb or delta) m= compute_md5(open(TD+'NEW.file')) if params['NEW/MD5sum'] != m : - fore() - raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) ) + f=fore() + raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta) elif this_deb_format == 'unzipped' : if DO_MD5: m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for control.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) m=compute_md5(subprocess.Popen('ar p "%s" data.tar' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/data.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for data.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for control.tar' ) + raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for data.tar' ) + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) @@ -1867,7 +2021,7 @@ if T : rmtree(T) return r -def do_delta_(olddeb,newdeb,delta,TD): +def do_delta_(olddeb,newdeb,delta,TD,forensic_file=None): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, @@ -1973,7 +2127,7 @@ self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') - self.fd.write('lzma') + self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') self.fd.write('xz') @@ -2559,7 +2713,7 @@ time_corr=0 #################### vvv delta_tar vvv ########################### - def delta_tar(old_filename, new_filename, CWD,\ + def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" @@ -2580,7 +2734,10 @@ oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) - + if old_forensic != None: + #fixme : devices are not supported (but debian policy does not allow them) + old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ + oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname @@ -2605,11 +2762,19 @@ if oldname in skip: if VERBOSE > 2 : print ' skipping ',repr(oldname) + if old_forensic != None: + oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + if old_forensic != None: + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) + oldtar.close() if type(old_filename) == StringType : unlink(TD+old_filename) @@ -2914,6 +3079,13 @@ ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') + if forensic_file==None: + control_forensic=None + data_forensic=None + else: + control_forensic=[] + data_forensic=[] + for name in ar_list_new : newname = 'NEW/'+name system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') @@ -2951,7 +3123,7 @@ if a not in dpkg_keeps_controls: skip.append(a) #delta it - delta_tar(oldname,newname,'CONTROL',skip) + delta_tar(oldname,newname,'CONTROL',control_forensic,skip) script.end_member() elif not NEEDSOLD and name[:8] == 'data.tar' : script.start_member(ar_line, newname, extrachar) @@ -2970,7 +3142,7 @@ def x(): return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.xz | unxz -c') else: assert(0) - delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\ + delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x script.end_member() @@ -3013,6 +3185,9 @@ #script is done script.close() + if forensic_file: + forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) + patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): @@ -3500,8 +3675,25 @@ deltatmp=delta+'_tmp_' ret= None tdir=tempo() + + forensicfile=None + if FORENSICDIR: + if 'Filename' in new: + forensicdirname=delta_dirname(os.path.dirname(new['Filename']),FORENSICDIR) + elif 'File' in new: + forensicdirname=delta_dirname(os.path.dirname(new['File']),FORENSICDIR) + else: + assert(0) + if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act + os.makedirs(forensicdirname) + forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' + a=os.path.join(forensicdirname,forensicbasename) + if not os.path.exists(a): + forensicfile=open(a,'w') + del a + try: - ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir) + ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir, forensic_file=forensicfile) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret except KeyboardInterrupt: if os.path.exists(deltatmp): @@ -3809,7 +4001,8 @@ # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError, s: - if s.errno == 11 : + from errno import EAGAIN + if s.errno == EAGAIN : a=' already locked!' else: a=str(s) @@ -3836,7 +4029,7 @@ patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch - def thread_do_patch(que, no_delta, returns, exitcodes): + def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : print ' Patching thread started. ' debs_size=0 debs_time=0 @@ -3876,6 +4069,7 @@ if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : print ' No deb-policy "e", no download of ',deb_uri + forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return @@ -4170,10 +4364,11 @@ ###################################### end of HTTP stuff ################### start patching thread + forensics=[] patching_thread=threading.Thread( target=thread_do_patch , - args=(patching_queue, no_delta, thread_returns, mainexitcodes) ) + args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() @@ -4453,7 +4648,7 @@ while patching_thread.isAlive(): time.sleep(0.1) - + #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.isAlive(): @@ -4486,7 +4681,9 @@ t=total_time print ' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )} - + + if forensics: + forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff @@ -4535,6 +4732,8 @@ raise SystemExit(5) except DebDeltaError,s: puke('debpatch',s) + if s.logs: + forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception,s: puke('debpatch',s) diff -Nru debdelta-0.44/debpatch-url debdelta-0.45/debpatch-url --- debdelta-0.44/debpatch-url 2011-08-25 11:05:53.000000000 +0000 +++ debdelta-0.45/debpatch-url 2011-12-06 16:22:44.000000000 +0000 @@ -169,6 +169,21 @@ DEB_POLICY = ['b','s','e'] DO_PROGRESS = terminalcolumns != None +#where/how debpatch/debdelta-upgrade will send forensic data, when patching fails +#possible values: +# False : do not send them +# True : compute forensic but not send them, just list them +# mail : automatically send by email to default address +# user@domain : automatically send by email to address +# mailto:user@domain : as above +# mutt:user@domain : as above, but use 'mutt', so the user can customize it +# http://domain/cgi : send them automatically thru a CGI script +#Warning: the above is mostly TODO +FORENSIC=False + +#directory tree where forensic info are stored by 'debdeltas' +FORENSICDIR=None + DEB_FORMAT='deb' DEB_FORMAT_LIST=('deb','unzipped','preunpacked') #not yet implemented on patching side : (,'piped') @@ -219,7 +234,7 @@ try: ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:A' , ('help','info','needsold','dir=','no-act','alt=','old=','delta-algo=', - 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug', + 'max-percent=','deb-policy=','clean-deltas','clean-alt','no-md5','debug','forensicdir=','forensic=', 'signing-key=', "accept-unsigned", "gpg-home=", "disable-feature=", "test", "format=") ) except getopt.GetoptError,a: sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n') @@ -260,6 +275,24 @@ if not os.path.isdir(DIR): sys.stderr.write( _("Error: argument of --dir is not a directory:") +' '+ DIR +'\n') raise SystemExit(3) + + elif o == '--forensicdir' : + FORENSICDIR = abspath(expanduser(v)) + if v[-2:] == '//': + FORENSICDIR += '//' + if not os.path.isdir(FORENSICDIR): + sys.stderr.write( _("Error: argument of --forensicdir is not a directory:") +' '+ FORENSICDIR +'\n') + raise SystemExit(3) + + elif o == '--forensic' : + FORENSIC = v + if FORENSIC[:4] == 'http': + try: + import poster + except: + print 'To use the http forensic, you must install the package "python-poster".' + raise SystemExit(3) + elif o == '--alt' : if not (os.path.isfile(v) or os.path.isdir(v)) : sys.stderr.write(_('Error: argument of --alt is not a directory or a regular file:')+' '+v +'\n') @@ -410,7 +443,9 @@ def de_bar(a): if a and a[:2] == './' : a=a[2:] - if a and a[0] == '/' : + elif a == '/.' : + a='' + elif a and a[0] == '/' : a=a[1:] return a @@ -546,7 +581,7 @@ class DebDeltaError(Exception): #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html # Subclasses that define an __init__ must call Exception.__init__ # or define self.args. Otherwise, str() will fail. - def __init__(self,s,retriable=False,exitcode=None): + def __init__(self,s,retriable=False,exitcode=None,logs=None): assert(type(s) == StringType) self.retriable = retriable if retriable: @@ -559,6 +594,7 @@ else: exitcode = 2 self.exitcode=exitcode + self.logs=logs def die(s): #if s : sys.stderr.write(s+'\n') @@ -1116,43 +1152,132 @@ a=a+ ( '%02x' % ord(i) ) return a -def forensics_rfc(o,db,controlfiles,files,diverted,diversions,conffiles=[]): - o.write('Package: '+db['OLD/Package']+'\n') - o.write('Version: '+db['OLD/Version']+'\n') - o.write('Architecture: '+db['OLD/Architecture']+'\n') +def forensics_rfc(o,db,bytar,controlfiles,files,conffiles,diverted=[],diversions={},localepurged=[],prelink_u_failed=[]): + " this is invoked by do_patch_() as well as do_delta_() ; in the former case, by_tar=False" + assert type(diversions) == dict + if type(db) == dict: + for a in sorted(db.keys()): + if a[:3] == 'OLD': + o.write(a[4:]+': '+db[a]+'\n') + else: + for a in sorted(db): + if a[:3] == 'OLD': + o.write(a[4:]+'\n') if diverted: o.write("Diversions:\n") - for a in diverted: + for a in sorted(diverted): b,p = diversions[a] o.write(" From: "+a+'\n') o.write(" To: "+b+'\n') o.write(" By: "+p+'\n') if conffiles: o.write("Conffiles:\n") - for a in conffiles: + for a in sorted(conffiles): o.write(' '+a+'\n') for L,N in ((controlfiles,"Control"),(files,"Files")): o.write(N+":\n") - for a,b in L: - if not os.path.exists(b): - o.write(' NONEXISTENT\n '+b+'\n \n') + for l in sorted(L): + if bytar: + name,mode,tartype,uid,gid,uname,gname,data=l + tmpcopy=None + divert=None + else: + name,divert,tmpcopy=l + if os.path.exists(divert): + fullname,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(divert) + else: + fullname,mode,tartype,uid,gid,uname,gname,data='',0,'?',0,0,'?','?','?' + if tartype == tarfile.REGTYPE: + if tmpcopy and os.path.exists(tmpcopy): + data=hash_to_hex(sha1_hash_file(tmpcopy)) + elif os.path.exists(divert): + data=hash_to_hex(sha1_hash_file(divert)) + if name in ('.', '/', './', '/.') and tartype == tarfile.DIRTYPE: #skip root continue - name,mode,tartype,uid,gid,uname,gname,data=stat_to_tar(b) - if tartype == tarfile.REGTYPE: - data=hash_to_hex(sha1_hash_file(b)) if uname == None: uname=str(uid) if gname == None: gname=str(gid) + name=de_bar(name) o.write(' '+tarinfo_to_ls(tartype,mode)+" "+uname+' '+gname) - if N == "Files" and tartype == tarfile.REGTYPE and a in conffiles: - o.write(" [conffile]\n") - else: - o.write("\n") - o.write(" "+a+"\n") + if N == "Files" and tartype == tarfile.REGTYPE and name in conffiles: + o.write(" [conffile]") + if N == "Files" and tartype == tarfile.REGTYPE and name in localepurged: + o.write(" [localpurged]") + if N == "Files" and tartype == tarfile.REGTYPE and name in prelink_u_failed: + o.write(" [prelink-u failed]") + if divert and not os.path.exists(divert): + o.write(" [missing file %r]" % divert) + if tmpcopy: + o.write(" [prelink-u]") + o.write("\n "+name+"\n") if data!=None: o.write(" "+data+"\n") else: o.write(" \n") +def forensic_send(f,forensic=FORENSIC): + " note that f must be a list of lists (or None)" + assert type(f) == list + if not forensic : + if f: + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + return + if not f: + return + if all([(z == None) for z in f]): + print 'Sorry, no forensic logs were generated' + return + if forensic[:4] in ('mutt','mail') or forensic[:7] == 'icedove' or forensic[:10]=='thunderbird': + email=EMAIL + if ':' in forensic: + a=forensic.find(':') + email == forensic[a:] + forensic=forensic[:a] + print _("There were faulty deltas.")+' '+_("Now invoking the mail sender to send the logs.") + if forensic in ('mutt','mail'): + raw_input( _('(hit any key)') ) + args=[] + for z in f: + if z: + for j in z: + args+=['-a',j] + subprocess.call(['mutt',email,'-s','delta_failures']+args) + else: + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + args="to=%s,subject=delta_failures,attachment='file:///%s'" % (email,temptar) + subprocess.call([forensic,'-compose',args]) + return + elif forensic[:4] == 'http': + print _("There were faulty deltas.")+' '+_('Sending logs to server.') + temptar=tempfile.mktemp(suffix='.tgz') + tar=tarfile.open(name=temptar,mode='w:gz') + for z in f: + if z: + for j in z: + tar.add(j,arcname=os.path.basename(j)) + tar.close() + #http://atlee.ca/software/poster + import urllib, urllib2, httplib, poster + poster.streaminghttp.register_openers() + datagen, headers = poster.encode.multipart_encode({'auth_userid':'debdelta','auth_password':'slartibartfast',"thefile": open(temptar, "rb")}) + # Create the Request object + request = urllib2.Request("http://debdelta.debian.net:7890/receive", datagen, headers) + # Actually do the request, and get the response + print ' '+_('Server answers:'),repr(urllib2.urlopen(request).read()) + return + else: + sys.stderr.write(_('Faulty delta. Please send by email to %s the following files:\n') % EMAIL) + for z in f: + if z: + sys.stderr.write(' '+string.join(z,' ')+'\n') + return + sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "--forensic=http" ).')+'\n') + def elf_info(f): "returns (is_elf, ei_class, ei_data, ei_osabi, e_type)" import struct @@ -1302,11 +1427,11 @@ #this is not needed in preparing the patch, but may help in forensic conf_files=[] - a='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' - if DEBUG and os.path.isfile(a): - #note that filenames have leading / - conf_files=[p for p in open(a).read().split('\n') if p] - del a + z='/var/lib/dpkg/info/'+params['OLD/Package']+'.conffiles' + if DEBUG and os.path.isfile(z): + #note that filenames do not have leading / + conf_files=[de_bar(p) for p in open(z).read().split('\n') if p] + del z ### s=patch_check_tmp_space(params,olddeb) @@ -1350,6 +1475,7 @@ elif params[a] != dpkg_params[a] : die( 'Error : in delta , '+a+' = ' +params[a] +\ '\nin old/installed deb, '+a+' = ' +dpkg_params[a]) + del b,p #cannot delete 'a', python raise a SyntaxError runtime['patchprogress']=5 @@ -1399,9 +1525,10 @@ p.close() return s, diverted - localepurged=[] - prelink_u_failed=[] def _symlink_data_tree(pa,TD,diversions,runtime): + localepurged=[] + prelink_u_failed=[] + file_triples=[] prelink_time = 0 prelink_datasize = 0 if diversions: @@ -1417,8 +1544,8 @@ if do_progress: sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) if os.path.isfile(divert) and not os.path.islink(divert) : - a=TD+'OLD/DATA'+orig - d=os.path.dirname(a) + tmpcopy=TD+'OLD/DATA'+orig + d=os.path.dirname(tmpcopy) if not os.path.exists(d): os.makedirs(d) #the following code idea was provided by roman@khimov.ru @@ -1433,40 +1560,54 @@ prelink_time -= time.time() prelink_datasize += os.path.getsize(divert) if VERBOSE > 3 : - print ' copying/unprelinking ',divert,' to ',a + print ' copying/unprelinking ',divert,' to ', tmpcopy #unfortunately 'prelink -o' sometimes alters files, see http://bugs.debian.org/627932 - shutil.copy2(divert,a) - proc=subprocess.Popen(["/usr/sbin/prelink","-u",a],stdin=open(os.devnull),\ + shutil.copy2(divert, tmpcopy) + proc=subprocess.Popen(["/usr/sbin/prelink","-u",tmpcopy],stdin=open(os.devnull),\ stdout=subprocess.PIPE,stderr=subprocess.STDOUT) out=proc.stdout.read().strip() proc.wait() if proc.returncode: - if not os.path.exists(a): - if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',a,')' - os.symlink(divert, a) + if not os.path.exists(tmpcopy): + if VERBOSE > 4 : print ' (prelink failed, symlinking ',divert,' to ',tmpcopy,')' + os.symlink(divert, tmpcopy) + prelink_u_failed.append(de_bar(orig)) + unprelink=False elif VERBOSE > 4 : print ' (prelink failed, but file was copied)' - thestat = os.statvfs(a) + thestat = os.statvfs(tmpcopy) if out[-39:] == 'does not have .gnu.prelink_undo section': if DEBUG: sys.stderr.write('!!'+repr(out)+'\n') elif (thestat.f_bsize * thestat.f_bavail / 1024) < 50000 : sys.stderr.write('!!Prelink -u failed, it needs at least 50000KB of free disk space\n') - prelink_u_failed.append(a) + prelink_u_failed.append(de_bar(orig)) + unprelink=False else: - sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (a,out)) - prelink_u_failed.append(a) + sys.stderr.write('!!Prelink -u failed on %s : %s\n' % (tmpcopy,out)) + prelink_u_failed.append(de_bar(orig)) + unprelink=False prelink_time += time.time() else: if VERBOSE > 3 : print ' symlinking ',divert,' to ',a - os.symlink(divert, a) + os.symlink(divert, tmpcopy) + if unprelink and FORENSIC: + #unfortunately the script will delete the 'tmpcopy', so we hardlink it + z=tempfile.mktemp(prefix=TD) + os.link(tmpcopy,z) + file_triples.append((orig,divert,z)) + else: + file_triples.append((orig,divert,None)) elif not os.path.exists(divert): + file_triples.append((orig,divert,None)) if VERBOSE : print ' Disappeared file? ',divert for z in ('locale','man','gnome/help','omf','doc/kde/HTML'): w='/usr/share/'+z if orig[:len(w)] == w: - localepurged.append(orig) - elif VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig - return s,diverted, prelink_time, prelink_datasize + localepurged.append(de_bar(orig)) + else: + file_triples.append((orig,divert,None)) + if VERBOSE > 3 : print ' not symlinking ',divert,' to ',orig + return file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize def chmod_add(n,m): "same as 'chmod ...+... n '" @@ -1487,8 +1628,14 @@ i=os.path.join(dirpath,i) chmod_add(i, S_IRUSR | S_IWUSR| S_IXUSR ) - control_file_pairs=[] - linked_file_pairs,diverted=[],[] + #initialize, just in case + control_file_triples=[] + file_triples=[] + localepurged=[] + prelink_u_failed=[] + diverted=[] + prelink_time=0 + prelink_datasize=0 ###see into parameters: the patch may need extra info and data @@ -1503,7 +1650,8 @@ elif 'old-data-tree' == a : os.mkdir(TD+'/OLD/DATA') if olddeb == '/': - linked_file_pairs,diverted,prelink_time,prelink_datasize=_symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) + file_triples, localepurged, prelink_u_failed, diverted, prelink_time, prelink_datasize=\ + _symlink_data_tree(params['OLD/Package'],TD,diversions,runtime) else: ar_list_old= list_ar(TD+'OLD.file') if 'data.tar.bz2' in ar_list_old: @@ -1526,10 +1674,11 @@ os.mkdir(TD+'OLD/CONTROL') p=params['OLD/Package'] for b in dpkg_keeps_controls : - a='/var/lib/dpkg/info/' + p +'.'+b - if os.path.exists(a): - os.symlink(a,TD+'OLD/CONTROL/'+b) - control_file_pairs.append((b,a)) + z='/var/lib/dpkg/info/' + p +'.'+b + if os.path.exists(z): + os.symlink(z,TD+'OLD/CONTROL/'+b) + control_file_triples.append((b,z,None)) + del z,p #cannot delete 'a', python raise a SyntaxError #else... we always unpack the control of a .deb elif 'needs-xdelta3' == a: if not os.path.exists('/usr/bin/xdelta3'): @@ -1563,8 +1712,6 @@ runtime['patchprogress']=12 - a='' - if DEBUG: a='-v' script_time = - time.time() this_deb_format=DEB_FORMAT @@ -1583,11 +1730,11 @@ elif this_deb_format == 'preunpacked': cmd+=['piped'] + env={'PATH':os.getenv('PATH')} F=subprocess.Popen(cmd, cwd=TD, - stdin=open(os.devnull), - stderr=subprocess.PIPE, stdout=temp_name_fd) - progresschar=0.0 - progresslen=float(os.path.getsize(os.path.join(TD,'PATCH/patch.sh'))) + bufsize=4096,close_fds=True, + stdin=open(os.devnull),env=env, + stderr=temp_err_name_fd, stdout=temp_name_fd) ### data used by the preunpacked method data_md5=None # md5 of uncompressed data.tar @@ -1690,13 +1837,19 @@ do_cleanup() raise else: #progress reporting for deb_format != 'preunpacked' - for j in F.stderr: - os.write(temp_err_name_fd, j) - progresschar+=len(j) - progress=(int(12.0 + 84.0 * progresschar / progresslen)) - runtime['patchprogress']=progress - if do_progress: - sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) + runtime['patchprogress']=12 + if 'NEW/Size' in params: + NEW_size=int(params['NEW/Size']) + while None == F.poll(): + if os.path.exists(TD+'NEW.file'): + a=os.path.getsize(TD+'NEW.file') + progress=(int(12.0 + 84.0 * a / NEW_size)) + else: + progress=12 + runtime['patchprogress']=progress + time.sleep(0.1) + if do_progress: + sys.stderr.write('P %2d%% %s\r' % (progress, newdebshortname)) F.wait() if do_progress and terminalcolumns: #clean up sys.stderr.write(' ' * terminalcolumns + '\r') @@ -1708,23 +1861,25 @@ runtime['patchprogress']=97 #helper for debugging - def tempos(): + def tempos(f): if os.path.getsize(temp_name): - sys.stderr.write('!! '+temp_name+'\n') + f.append(temp_name) if os.path.getsize(temp_err_name): - sys.stderr.write('!! '+temp_err_name+'\n') + f.append(temp_err_name) - if DEBUG == 0: + if not FORENSIC: def fore(): - sys.stderr.write(_('(Faulty delta. Please consider retrying with the option "-d" ).')+'\n') + return None elif olddeb != '/': def fore(): - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! '+delta+'\n!! '+olddeb+'\n') - tempos() + f=[delta,olddeb] + tempos(f) + return f else: def fore(): temp_fore_name='' + f=[] + tempos(f) try: (temp_fd,temp_fore_name) = tempfile.mkstemp(prefix="debforensic_") temp_file=os.fdopen(temp_fd,'w') @@ -1732,74 +1887,73 @@ temp_file.write('DeltaSHA1: '+hash_to_hex(sha1_hash_file(delta))+'\n') temp_file.write('LocalePurgedFilesN: '+str(len(localepurged))+'\n') temp_file.write('PrelinkUFailedN: '+str(len(prelink_u_failed))+'\n') + for i in f: + temp_file.write('PatchLogFile: '+str(i)+'\n') if ret: temp_file.write('PatchExitCode: '+str(ret)+'\n') - forensics_rfc(temp_file,params,control_file_pairs, - linked_file_pairs,diverted,diversions,conf_files) + forensics_rfc(temp_file,params,False,control_file_triples,file_triples,conf_files, + diverted,diversions,localepurged,prelink_u_failed) temp_file.close() except OSError: #Exception,s: die('!!While creating forensic '+temp_fore_name+' error:'+str(s)+'\n') - sys.stderr.write('!!'+_('Faulty delta. Please send by email to %s the following files:') % EMAIL +\ - '\n!! ' + temp_fore_name + '\n') - tempos() - - ##then , really execute the patch + f.append(temp_fore_name) + return f if ret: if localepurged: raise DebDeltaError('"debdelta" is incompatible with "localepurge".') else: - fore() - raise DebDeltaError('error in patch.sh.') + f=fore() + raise DebDeltaError('error in patch.sh.',logs=f) #then we check for the conformance if this_deb_format == 'deb': if 'NEW/Size' in params: newdebsize = os.stat(TD+'NEW.file')[ST_SIZE] if newdebsize != int(params['NEW/Size']): - fore() - raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size']) + f=fore() + raise DebDeltaError('new deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'],logs=f) if DO_MD5: if 'NEW/MD5sum' in params: if VERBOSE > 1 : print ' verifying MD5 for ',os.path.basename(newdeb or delta) m= compute_md5(open(TD+'NEW.file')) if params['NEW/MD5sum'] != m : - fore() - raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) ) + f=fore() + raise DebDeltaError(' MD5 mismatch, '+repr(params['NEW/MD5sum'])+' != ' + repr(m) , logs=f) else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta) elif this_deb_format == 'unzipped' : if DO_MD5: m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for control.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for control.tar' , logs=f) m=compute_md5(subprocess.Popen('ar p "%s" data.tar' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/data.tar'][:32] != m: - fore() - raise DebDeltaError('MD5 mismatch for data.tar' ) + f=fore() + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) elif this_deb_format == 'preunpacked' : if tar_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in tar: "+repr(tar_status[0][1]), logs=f) #todo format me better if md5_status != [True]: - fore() + f=fore() do_cleanup() - raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1])) #todo format me better + raise DebDeltaError("something bad happened in md5: "+repr(md5_status[0][1]), logs=f) #todo format me better #if DO_MD5: #actually we always do MD5 m=compute_md5(subprocess.Popen('ar p "%s" control.tar.gz | zcat' % (TD+'NEW.file'), stdout=subprocess.PIPE,shell=True).stdout) if params['NEW/control.tar'][:32] != m: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for control.tar' ) + raise DebDeltaError('MD5 mismatch for control.tar', logs=f) if params['NEW/data.tar'][:32] != data_md5: - fore() + f=fore() do_cleanup() - raise DebDeltaError('MD5 mismatch for data.tar' ) + raise DebDeltaError('MD5 mismatch for data.tar', logs=f) else: assert('unimplemented'=='') os.unlink(temp_name) @@ -1867,7 +2021,7 @@ if T : rmtree(T) return r -def do_delta_(olddeb,newdeb,delta,TD): +def do_delta_(olddeb,newdeb,delta,TD,forensic_file=None): """This function creates a delta. The delta is 'ar' archive (see 'man ar'). The delta contains data, a script, and optional gpg signatures. The script recreates the new deb. Note that the deb is (again) an 'ar' archive, @@ -1973,7 +2127,7 @@ self.fd.write('./minibzip2 -9') elif cn == '.lzma' : info_append('needs-lzma') - self.fd.write('lzma') + self.fd.write('lzma -9') elif cn == '.xz' : info_append('needs-xz') self.fd.write('xz') @@ -2559,7 +2713,7 @@ time_corr=0 #################### vvv delta_tar vvv ########################### - def delta_tar(old_filename, new_filename, CWD,\ + def delta_tar(old_filename, new_filename, CWD, old_forensic,\ skip=[], old_md5={}, new_md5={},\ chunked_p=(not delta_uses_infifo) ,debdelta_conf_skip=()): " compute delta of two tar files, and prepare the script consequently" @@ -2580,7 +2734,10 @@ oldtarinfos = {} for oldtarinfo in oldtar: oldname = de_bar(oldtarinfo.name) - + if old_forensic != None: + #fixme : devices are not supported (but debian policy does not allow them) + old_forensic.append([oldtarinfo.name,oldtarinfo.mode,oldtarinfo.type,\ + oldtarinfo.uid,oldtarinfo.gid,oldtarinfo.uname,oldtarinfo.gname,oldtarinfo.linkname]) #this always happens #if VERBOSE > 3 and oldname != de_bar(oldname): # print ' filename in old tar has weird ./ in front: ' , oldname @@ -2605,11 +2762,19 @@ if oldname in skip: if VERBOSE > 2 : print ' skipping ',repr(oldname) + if old_forensic != None: + oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) continue oldnames.append(oldname) oldtarinfos[oldname] = oldtarinfo oldtar.extract(oldtarinfo,TD+"OLD/"+CWD ) + if old_forensic != None: + old_forensic.append(old_forensic.pop()[:-1] + \ + [hash_to_hex(sha1_hash_file(os.path.join(TD,"OLD",CWD,oldname)))]) + oldtar.close() if type(old_filename) == StringType : unlink(TD+old_filename) @@ -2914,6 +3079,13 @@ ar_list_old= list_ar(TD+'OLD.file') ar_list_new= list_ar(TD+'NEW.file') + if forensic_file==None: + control_forensic=None + data_forensic=None + else: + control_forensic=[] + data_forensic=[] + for name in ar_list_new : newname = 'NEW/'+name system(('ar','x',TD+'NEW.file',name), TD+'/NEW/') @@ -2951,7 +3123,7 @@ if a not in dpkg_keeps_controls: skip.append(a) #delta it - delta_tar(oldname,newname,'CONTROL',skip) + delta_tar(oldname,newname,'CONTROL',control_forensic,skip) script.end_member() elif not NEEDSOLD and name[:8] == 'data.tar' : script.start_member(ar_line, newname, extrachar) @@ -2970,7 +3142,7 @@ def x(): return my_popen_read('cd '+TD+'; ar p OLD.file data.tar.xz | unxz -c') else: assert(0) - delta_tar(x,newname,'DATA',old_conffiles,old_md5,new_md5,\ + delta_tar(x,newname,'DATA',data_forensic,old_conffiles,old_md5,new_md5,\ debdelta_conf_skip=debdelta_conf_skip) del x script.end_member() @@ -3013,6 +3185,9 @@ #script is done script.close() + if forensic_file: + forensics_rfc(forensic_file,info,True,control_forensic,data_forensic,old_conffiles) + patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE] patch_files = [] if 'lzma' not in DISABLED_FEATURES and os.path.exists('/usr/bin/lzma'): @@ -3500,8 +3675,25 @@ deltatmp=delta+'_tmp_' ret= None tdir=tempo() + + forensicfile=None + if FORENSICDIR: + if 'Filename' in new: + forensicdirname=delta_dirname(os.path.dirname(new['Filename']),FORENSICDIR) + elif 'File' in new: + forensicdirname=delta_dirname(os.path.dirname(new['File']),FORENSICDIR) + else: + assert(0) + if not os.path.exists(forensicdirname): #FIXME this does not respect --no-act + os.makedirs(forensicdirname) + forensicbasename = pa +'_'+ version_mangle(old['Version']) +'_'+ar+'.forensic' + a=os.path.join(forensicdirname,forensicbasename) + if not os.path.exists(a): + forensicfile=open(a,'w') + del a + try: - ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir) + ret=do_delta_(old['File'],new['File'], deltatmp, TD=tdir, forensic_file=forensicfile) (deltatmp_, percent, elaps, info_delta, gpg_hashes) = ret except KeyboardInterrupt: if os.path.exists(deltatmp): @@ -3809,7 +4001,8 @@ # synopsis lockf( fd, operation, [length, [start, [whence]]]) fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0) except IOError, s: - if s.errno == 11 : + from errno import EAGAIN + if s.errno == EAGAIN : a=' already locked!' else: a=str(s) @@ -3836,7 +4029,7 @@ patching_queue=Queue.Queue() thread_returns={} ######################## thread_do_patch - def thread_do_patch(que, no_delta, returns, exitcodes): + def thread_do_patch(que, no_delta, returns, exitcodes, forensics): if VERBOSE > 1 : print ' Patching thread started. ' debs_size=0 debs_time=0 @@ -3876,6 +4069,7 @@ if 'e' in DEB_POLICY: no_delta.append( (deb_uri, newdeb) ) elif VERBOSE > 1 : print ' No deb-policy "e", no download of ',deb_uri + forensics.append(s.logs) exitcodes.append(s.exitcode) except: if puke == None: return @@ -4170,10 +4364,11 @@ ###################################### end of HTTP stuff ################### start patching thread + forensics=[] patching_thread=threading.Thread( target=thread_do_patch , - args=(patching_queue, no_delta, thread_returns, mainexitcodes) ) + args=(patching_queue, no_delta, thread_returns, mainexitcodes, forensics) ) patching_thread.daemon=True patching_thread.start() @@ -4453,7 +4648,7 @@ while patching_thread.isAlive(): time.sleep(0.1) - + #terminate progress report thread_returns['STOP']=True while progress_thread != None and progress_thread.isAlive(): @@ -4486,7 +4681,9 @@ t=total_time print ' ' + _('total resulting debs, size %(size)s time %(time)dsec virtual speed %(speed)s/sec') % \ {'size' : SizeToKibiStr(a), 'time' : int(t), 'speed' : SizeToKibiStr(a / t )} - + + if forensics: + forensic_send(forensics) return max(mainexitcodes) ################################################# main program, do stuff @@ -4535,6 +4732,8 @@ raise SystemExit(5) except DebDeltaError,s: puke('debpatch',s) + if s.logs: + forensic_send([s.logs]) raise SystemExit(s.exitcode) except Exception,s: puke('debpatch',s)