1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """
16 A user friendly interface to access all the function associated to MadWeight
17 """
18
19 from __future__ import absolute_import
20 import logging
21 import os
22 import subprocess
23 import time
24 import glob
25 import math
26 import xml.sax.handler
27 import shutil
28 from six import StringIO
29 from six.moves import map
30 from six.moves import range
31
32 if __name__ == '__main__':
33 import sys
34 sys.path.append('/Users/omatt/Documents/eclipse/madweight/')
35
36 logger = logging.getLogger('cmdprint')
37
38 pjoin = os.path.join
39 try:
40 from madgraph import InvalidCmd, MadGraph5Error, MG5DIR
41 import madgraph.interface.extended_cmd as cmd
42 import madgraph.interface.common_run_interface as common_run
43
44
45 import madgraph.madweight.MW_info as MW_info
46 import madgraph.madweight.change_tf as change_tf
47 import madgraph.madweight.create_param as create_param
48 import madgraph.madweight.create_run as create_run
49 import madgraph.madweight.Cards as Cards
50 import madgraph.madweight.write_MadWeight as write_MadWeight
51 import madgraph.madweight.verif_event as verif_event
52 import madgraph.madweight.MW_driver as MW_driver
53
54 import madgraph.various.misc as misc
55 import madgraph.various.banner as banner
56 import madgraph.iolibs.files as files
57 MADEVENT = False
58 except ImportError as error:
59 raise
60 logger.debug(error)
61 from internal import InvalidCmd, MadGraph5Error
62 import internal.extended_cmd as cmd
63 import internal.common_run_interface as common_run
64 import internal.madweight.MW_info as MW_info
65 import internal.madweight.change_tf as change_tf
66 import internal.madweight.create_param as create_param
67 import internal.madweight.create_run as create_run
68 import internal.madweight.Cards as Cards
69 import internal.madweight.write_MadWeight as write_MadWeight
70 import internal.madweight.verif_event as verif_event
71 import internal.madweight.MW_driver as MW_driver
72
73
74 import internal.misc as misc
75 import internal.banner as banner
76 import internal.files as files
77 MADEVENT = True
78
79
80 AlreadyRunning = common_run.AlreadyRunning
81
82
83
84
86 """Particularisation of the cmd command for MadEvent"""
87
88
89 next_possibility = {
90 'start': [],
91 }
92
93 debug_output = 'MW5_debug'
94 error_debug = 'Please report this bug on https://bugs.launchpad.net/mg5amcnlo\n'
95 error_debug += 'with MadWeight in the title of the bug report.'
96 error_debug += 'More information is found in \'%(debug)s\'.\n'
97 error_debug += 'Please attach this file to your report.'
98
99 config_debug = 'If you need help with this issue please contact us on https://answers.launchpad.net/mg5amcnlo\n'
100
101
102 keyboard_stop_msg = """stopping all operation
103 in order to quit madweight please enter exit"""
104
105
106 InvalidCmd = InvalidCmd
107 ConfigurationError = MadGraph5Error
108
110 """Init history and line continuation"""
111
112
113 self.force = False
114
115
116
117 info = misc.get_pkg_info()
118 info_line = ""
119 if info and 'version' in info and 'date' in info:
120 len_version = len(info['version'])
121 len_date = len(info['date'])
122 if len_version + len_date < 30:
123 info_line = "#* VERSION %s %s %s *\n" % \
124 (info['version'],
125 (30 - len_version - len_date) * ' ',
126 info['date'])
127 else:
128 root_path = pjoin(os.path.dirname(__file__), os.path.pardir,os.path.pardir)
129 version = open(pjoin(root_path,'MGMEVersion.txt')).readline().strip()
130 info_line = "#* VERSION %s %s *\n" % \
131 (version, (24 - len(version)) * ' ')
132
133
134
135 self.history_header = \
136 '#************************************************************\n' + \
137 '#* MadWeight 5 *\n' + \
138 '#* *\n' + \
139 "#* * * *\n" + \
140 "#* * * * * *\n" + \
141 "#* * * * * 5 * * * * *\n" + \
142 "#* * * * * *\n" + \
143 "#* * * *\n" + \
144 "#* *\n" + \
145 "#* *\n" + \
146 info_line + \
147 "#* *\n" + \
148 "#* The MadGraph Development Team - Please visit us at *\n" + \
149 "#* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \
150 '#* *\n' + \
151 '#************************************************************\n' + \
152 '#* *\n' + \
153 '#* Command File for MadWeight *\n' + \
154 '#* *\n' + \
155 '#* run as ./bin/madweight.py FILENAME *\n' + \
156 '#* *\n' + \
157 '#************************************************************\n'
158
159 if info_line:
160 info_line = info_line[1:]
161
162 logger.info(\
163 "************************************************************\n" + \
164 "* *\n" + \
165 "* W E L C O M E to M A D G R A P H 5 *\n" + \
166 "* M A D W E I G H T *\n" + \
167 "* *\n" + \
168 "* * * *\n" + \
169 "* * * * * *\n" + \
170 "* * * * * 5 * * * * *\n" + \
171 "* * * * * *\n" + \
172 "* * * *\n" + \
173 "* *\n" + \
174 info_line + \
175 "* *\n" + \
176 "* The MadGraph Development Team - Please visit us at *\n" + \
177 "* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \
178 "* *\n" + \
179 "* Type 'help' for in-line help. *\n" + \
180 "* *\n" + \
181 "************************************************************")
182
183 cmd.Cmd.__init__(self, *arg, **opt)
184
186
188 logger.info('collect [-refine]')
189 logger.info(' combine the results of the jobs launched on the cluster.')
190 logger.info(' This creates three type of output files:')
191 logger.info(' - weights.out [weight for event/card]')
192 logger.info(' - unnormalize-likelihood.out [-\sum ln(Weight)]')
193 logger.info(' - output.xml [additional information]')
194 logger.info('')
195 logger.info(' The option \'-refine\' is to be added if this is not the first')
196 logger.info(' cluster submission. Otherwise previous run submission will be lost.')
197
199 """help for define transfer_fct"""
200 logger.info(' Modify the current transfer functions')
201 logger.info(' If no argument provided a question showing the list of possibilities.')
202 logger.info(' will be ask. If the TF is provided as argument, no question is asked.')
203
204
206
207
209 """ complete the collect command"""
210 args = self.split_arg(line[0:begidx])
211
212 return self.list_completion(text,['-refine','-f','--refine'], line)
213
215 """ complete the define_transfer_fct """
216
217 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data')
218 listdir=os.listdir(path)
219 args = self.split_arg(line[0:begidx])
220 if len(args) == 1:
221
222 possibilities = [content[3:-4] for content in listdir \
223 if (content.startswith('TF') and content.endswith('dat'))]
224 return self.list_completion(text, possibilities, line)
225
226
227
228
229 -class MadWeightCmd(CmdExtended, HelpToCmd, CompleteForCmd, common_run.CommonRunCmd):
230
231 _set_options = []
232 prompt = 'MadWeight5>'
233 helporder = ['MadWeight Function', 'Documented commands', 'Advanced commands']
234
236 """Not in help: remove fct"""
237 return None
238
239 do_decay_events = remove_fct
240 do_delphes = remove_fct
241 do_pgs = remove_fct
242 _True = [1,True,'1','T','t','.true.','True']
243
244
245 - def __init__(self, me_dir = None, options={}, *completekey, **stdin):
251
256
286
288 """MadWeight Function:Define the current transfer function"""
289
290 with misc.chdir(self.me_dir):
291 self.configure()
292 args = self.split_arg(line)
293
294 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data')
295 listdir=os.listdir(path)
296 question = 'Please choose your transfer_function between\n'
297 possibilities = [content[3:-4] for content in listdir \
298 if (content.startswith('TF') and content.endswith('dat'))]
299 for i, tfname in enumerate(possibilities):
300 question += ' %s / %s\n' % (i, tfname)
301 possibilities += list(range(len(possibilities)))
302
303 if args and args[0] in possibilities:
304 tfname = args[0]
305 else:
306 tfname = self.ask(question, 'dbl_gauss_pt_jet', possibilities)
307 if tfname.isdigit():
308 tfname = possibilities[int(tfname)]
309
310 P_dir, MW_dir = MW_info.detect_SubProcess(P_mode=1)
311 os.chdir('./Source/MadWeight/transfer_function')
312 change_tf.create_TF_main(tfname,0, MW_dir)
313
314
324
331
333 """MadWeight Function:compile the code"""
334 self.configure()
335
336 misc.compile(arg=["../lib/libtools.a"], cwd=pjoin(self.me_dir,'Source'))
337 misc.compile(arg=["../lib/libblocks.a"], cwd=pjoin(self.me_dir,'Source'))
338 misc.compile(arg=["../lib/libTF.a"], cwd=pjoin(self.me_dir,'Source'))
339 misc.compile(arg=["../lib/libpdf.a"], cwd=pjoin(self.me_dir,'Source'))
340 misc.compile(arg=["../lib/libdhelas.a"], cwd=pjoin(self.me_dir,'Source'))
341 misc.compile(arg=["../lib/libmodel.a"], cwd=pjoin(self.me_dir,'Source'))
342 misc.compile(arg=["../lib/libgeneric.a"], cwd=pjoin(self.me_dir,'Source'))
343 misc.compile(arg=["../lib/libcernlib.a"], cwd=pjoin(self.me_dir,'Source'))
344
345
346
347 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['montecarlo_perm']=='t':
348 raise Exception('Cannot use mint if monte carlo over permutations')
349 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['use_sobol']=='t':
350 raise Exception('sobol generator with mint not implemented')
351
352
353
354 for MW_dir in self.MWparam.MW_listdir:
355 logger.info('compile %s' %MW_dir)
356 pdir = pjoin(self.me_dir,'SubProcesses',MW_dir)
357 if refine and os.path.exists(pjoin(pdir, 'initialization.o')):
358 os.remove(pjoin(pdir, 'initialization.o'))
359 if refine and os.path.exists(pjoin(pdir, 'comp_madweight')):
360 os.remove(pjoin(pdir, 'comp_madweight'))
361 misc.compile(cwd=pdir)
362 if not os.path.exists(pjoin(pdir, 'comp_madweight')):
363 raise Exception('compilation fails')
364 logger.info('MadWeight code has been compiled.')
365
366
368 """MadWeight Function: check that the events are valid
369 and write the events to MG mapping"""
370 self.configure()
371 evt_file = pjoin(self.me_dir,'Events','input.lhco')
372 if not os.path.exists(evt_file):
373 question = 'Which LHCO file do you want to use?'
374 default = ''
375 if os.path.exists('%s.gz' % evt_file):
376 input_file = '%s.gz' % evt_file
377 else:
378 input_file = self.ask(question, default, path_msg='valid path')
379
380 if not input_file:
381 raise self.InvalidCmd('Please specify a valid LHCO File')
382
383 if input_file.endswith('.gz'):
384 misc.gunzip(input_file, keep=True, stdout=evt_file)
385 else:
386 files.cp(input_file, evt_file)
387
388 verif_event.verif_event(self.MWparam)
389
390
391
392
393
394
396 """format the argument to retrun a list with two argument,
397 The first corresponding to the fact if we need te create the output dir
398 The second if we can launch the job on the cluster."""
399
400 if not args:
401
402 args[:] = [True, True]
403 return
404 else:
405 create_dir = True
406 launch = True
407 for arg in args:
408 if arg.count('=') !=1 :
409 logger.warning('command launch_jobs does not recognized argument %s. This argument is ignored' % arg)
410 restrict, value = arg.split('=')
411 if restrict == '--create_dir=':
412 if value in self._True:
413 create_dir = True
414 else:
415 create_dir = False
416 elif restrict == '--submit=':
417 if value in self._True:
418 launch = True
419 else:
420 launch = False
421 args[:] = [create_dir, launch]
422 return
423
425 """MadWeight Function:Submitting the jobs to the cluster"""
426
427
428 self.configure()
429 self.clean_old_run(keep_event=True)
430 args = self.split_arg(line)
431 self.check_launch_jobs(args)
432
433 create_dir, launch_jobs = args[0], args[1]
434
435 for nb_card in self.MWparam.actif_param:
436 for dirname in self.MWparam.MW_listdir:
437 nb_job = self.MWparam.nb_event_MW[dirname]
438 if self.MWparam['mw_run']['nb_event_by_node'] > 1:
439 nb_job = 1+ (nb_job-1) // self.MWparam['mw_run']['nb_event_by_node']
440
441 for event_sample in range(nb_job):
442 self.submit_job(dirname, nb_card, event_sample)
443
444 starttime = time.time()
445
446 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'),
447 starttime=starttime, level='madweight', update_results=False)
448
449 try:
450 self.cluster.wait(self.me_dir, update_status)
451 except Exception:
452 self.cluster.remove()
453 raise
454 except KeyboardInterrupt:
455 if not self.force:
456 ans = self.ask('Error detected. Do you want to clean the queue?',
457 default = 'y', choices=['y','n'])
458 else:
459 ans = 'y'
460 if ans == 'y':
461 self.cluster.remove()
462 raise
463
464 - def submit_job(self, dirname, nb_card, sample_nb, evt_file=None, restrict_evt=[]):
465 """launch on the cluster the job which creates the computation"""
466
467 input_files = [pjoin(self.me_dir, 'SubProcesses', dirname, 'comp_madweight'),
468 pjoin(self.me_dir, 'Cards', 'param_card_%i.dat' % nb_card),
469 self.get_pdf_input_filename(),
470 pjoin(self.me_dir, 'Cards', 'ident_card.dat'),
471 pjoin(self.me_dir, 'Cards', 'run_card.dat')
472 ]
473
474
475 if not evt_file:
476 evt_file = (sample_nb // self.MWparam['mw_run']['event_packing'])
477 evt = 'verif_%i.lhco' % evt_file
478 first_event = (sample_nb % self.MWparam['mw_run']['event_packing']) * self.MWparam['mw_run']['nb_event_by_node']
479 name = self.MWparam.name
480 input_files.append(pjoin(self.me_dir, 'SubProcesses', dirname, name, evt))
481
482 if restrict_evt:
483 restrict_path = pjoin(self.me_dir, 'SubProcesses', dirname, name,
484 'restrict%i_%i.dat' % (nb_card,evt_file))
485 input_files.append(restrict_path)
486
487
488
489
490 output_file = ['output_%s_%s.xml' % (nb_card, sample_nb)]
491 exe = pjoin(self.me_dir, 'bin', 'internal', 'madweight', 'MW_driver.py')
492
493
494 args = [str(nb_card), str(first_event),
495 str(self.MWparam['mw_run']['nb_event_by_node']) ,evt,
496 str(self.MWparam['mw_run']['mw_int_points']),
497 self.MWparam['mw_run']['log_level'], str(sample_nb)]
498 cwd = pjoin(self.me_dir, 'SubProcesses', dirname, name)
499
500 if self.options['run_mode'] == 0:
501 raise Exception('need to check the validity')
502 else:
503
504 if not self.options['cluster_temp_path'] and not self.options['cluster_type'] == 'condor':
505 raise self.ConfigurationError('MadWeight requires temp_cluster_path options to be define')
506 self.cluster.submit2(exe, args, cwd, input_files=input_files, output_files=output_file)
507
508
509
522
524 """MadWeight Function: making the collect of the results"""
525
526 self.configure()
527 args = self.split_arg(line)
528 self.check_collect(args)
529 xml_reader = MWParserXML()
530
531 name = self.MWparam.name
532
533 out_dir = pjoin(self.me_dir, 'Events', name)
534 if '-refine' in args:
535 out_path = pjoin(out_dir, 'refine.xml')
536 else:
537 out_path = pjoin(out_dir, 'output.xml')
538 if os.path.exists(out_path):
539 logger.warning('Output file already exists. Current one will be tagged with _old suffix')
540 logger.warning('Run "collect -refine to instead update your current results."')
541 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
542 files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out'))
543 for MWdir in self.MWparam.MW_listdir:
544 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
545 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
546 out_dir = pjoin(self.me_dir, 'Events', name)
547
548 fsock = open(out_path, 'w')
549 fsock.write('<madweight>\n<banner>\n')
550
551 for card in ['proc_card_mg5.dat','MadWeight_card.dat','transfer_card.dat','param_card.dat','run_card.dat']:
552 cname = card[:-4]
553 fsock.write('<%s>\n' % cname)
554 fsock.write(open(pjoin(self.me_dir,'Cards',card)).read().replace('<','!>'))
555 fsock.write('</%s>\n' % cname)
556 fsock.write('</banner>\n')
557 at_least_one = False
558 for MWdir in self.MWparam.MW_listdir:
559 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
560 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name)
561 if not os.path.exists(out_dir):
562 os.mkdir(out_dir)
563 if '-refine' in args:
564 out_path = pjoin(out_dir, 'refine.xml')
565 else:
566 out_path = pjoin(out_dir, 'output.xml')
567 fsock2 = open(out_path,'w')
568 fsock.write('<subprocess id=\'%s\'>\n' % MWdir)
569 fsock2.write('<subprocess id=\'%s\'>\n' % MWdir)
570 for output in misc.glob('output_*_*.xml', input_dir):
571 at_least_one = True
572 text = open(output).read()
573 fsock2.write(text)
574 fsock.write(text)
575 os.remove(output)
576 fsock.write('</subprocess>\n')
577 fsock2.write('</subprocess>\n')
578 fsock2.close()
579 fsock.write('\n</madweight>\n')
580 fsock.close()
581
582 if '-refine' in args:
583 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level'])
584 for MWdir in self.MWparam.MW_listdir:
585 out_dir = pjoin(self.me_dir, 'Events',name, MWdir)
586 ref_output = xml_reader2.read_file(pjoin(out_dir, 'refine.xml'))
587 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level'])
588 base_output = xml_reader2.read_file(pjoin(out_dir, 'output.xml'))
589
590 base_output.refine(ref_output)
591 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
592 base_output.write(pjoin(out_dir, 'output.xml'), MWdir)
593 elif not at_least_one:
594 logger.warning("Nothing to collect restore _old file as current.")
595 out_dir = pjoin(self.me_dir, 'Events', name)
596 files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml'))
597 files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out'))
598 for MWdir in self.MWparam.MW_listdir:
599 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
600 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
601
602
603
604
605 total = {}
606 likelihood = {}
607 err_likelihood = {}
608 cards = set()
609 events = set()
610 tf_sets = set()
611 for MW_dir in self.MWparam.MW_listdir:
612 out_dir = pjoin(self.me_dir, 'Events', name, MW_dir)
613 xml_reader = MWParserXML()
614 data = xml_reader.read_file(pjoin(out_dir, 'output.xml'))
615
616 log_level = self.MWparam['mw_run']['log_level']
617 generator = ((int(i),j,int(k),data[i][j][k]) for i in data
618 for j in data[i]
619 for k in data[i][j])
620 for card, event, tf_set, obj in generator:
621
622 cards.add(card)
623 events.add(event)
624 tf_sets.add(tf_set)
625
626 if (card,event, tf_set) in total:
627 value, error = total[(card, event, tf_set)]
628 else:
629 value, error = 0, 0
630 obj.calculate_total()
631 value, error = (value + obj.value, error + obj.error**2)
632 total[(card, event, tf_set)] = (value, error)
633 if tf_set == 1:
634 if value:
635 if card not in likelihood:
636 likelihood[card], err_likelihood[card] = 0, 0
637 likelihood[card] -= math.log(value)
638 err_likelihood[card] += error / value
639 else:
640 likelihood[card] = float('Inf')
641 err_likelihood[card] = float('nan')
642
643
644
645 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w')
646 logger.info('Write output file with weight information: %s' % fsock.name)
647 fsock.write('# Weight (un-normalize) for each card/event/set of transfer fct\n')
648 fsock.write('# format: LHCO_event_number card_id tf_set_id value integration_error\n')
649 events = list(events)
650 events.sort()
651 cards = list(cards)
652 cards.sort()
653 tf_sets = list(tf_sets)
654 tf_sets.sort()
655 for event in events:
656 for card in cards:
657 for tf_set in tf_sets:
658 try:
659 value, error = total[(card, event,tf_set)]
660 except KeyError:
661 continue
662 error = math.sqrt(error)
663 fsock.write('%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error))
664
665
666 fsock = open(pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w')
667 fsock.write('# Warning: this Likelihood needs a bin by bin normalization !\n')
668 fsock.write('# IF more than one set of transfer function are define. ONLY the first one is ')
669 fsock.write('# include in this file.')
670 fsock.write('# format: card_id value integration_error\n')
671 for card in cards:
672 value, error = likelihood[card], err_likelihood[card]
673 error = math.sqrt(error)
674 fsock.write('%s %s %s \n' % (card, value, error))
675
676
678 """MadWeight Function: syntax: clean [XXX]
679 clean the previous run XXX (the last one by default)"""
680
681 args = self.split_arg(line)
682 self.configure()
683 if len(args) == 0:
684 name = self.MWparam.name
685 else:
686 name = args[0]
687
688 ans = self.ask('Do you want to remove Events/%s directory?', 'y',['y','n'])
689 if ans == 'y':
690 try:
691 shutil.rmtree(pjoin(self.me_dir, 'Events', name))
692 except Exception as error:
693 logger.warning(error)
694 for Pdir in self.MWparam.MW_listdir:
695 try:
696 shutil.rmtree(pjoin(self.me_dir, 'SubProcesses', Pdir, name))
697 except Exception as error:
698 logger.warning(error)
699
704
706 """MadWeight Function:run the full suite of commands"""
707
708 args = self.split_arg(line)
709
710
711
712
713 cards = ['param_card.dat', 'run_card.dat', 'madweight_card.dat',
714 'transfer_card.dat', 'input.lhco']
715 if not self.force:
716 self.ask_edit_cards(cards, mode='fixed', plot=False)
717 else:
718 self.configured = 0
719 self.configure()
720 with misc.chdir(self.me_dir):
721 if not os.path.exists(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile'])):
722 raise self.InvalidCmd('Please specify a valid LHCO File')
723 if pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']) not in \
724 [pjoin(self.me_dir, 'Events', 'input.lhco'), pjoin(self.me_dir, 'Events', 'input.lhco.gz')]:
725 zipped = self.MWparam['mw_run']['inputfile'].endswith('.gz')
726 if zipped:
727 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']),
728 pjoin(self.me_dir, 'Events', 'input.lhco.gz'))
729 if os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')):
730 os.remove(pjoin(self.me_dir, 'Events', 'input.lhco'))
731 else:
732 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']),
733 pjoin(self.me_dir, 'Events', 'input.lhco'))
734
735 if not (os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')) or \
736 os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco.gz'))):
737 raise self.InvalidCmd('Please specify a valid LHCO File')
738
739 self.exec_cmd('treatcards')
740 self.exec_cmd('get_integration_channel')
741 self.exec_cmd('compile')
742 self.exec_cmd('check_events')
743 self.exec_cmd('submit_jobs')
744 self.exec_cmd('collect')
745
746
748 """check the argument validity"""
749
750 if len(args) != 1:
751 raise self.InvalidCmd('refine requires a single argument')
752
753 try:
754 args[0] = float(args[0])
755 except Exception:
756 raise self.InvalidCmd('First argument of refine command should be a number')
757
758 if args[0] < 0 or args[0]>1:
759 raise self.InvalidCmd('The first argument should be a number between 0 and 1.')
760
762
763 for MWdir in self.MWparam.MW_listdir:
764 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, self.MWparam.name)
765 if not os.path.exists(input_dir):
766 continue
767 for filename in os.listdir(input_dir):
768 if keep_event and filename.startswith('verif'):
769 continue
770 os.remove(pjoin(input_dir, filename))
771
772
773
775 """MadWeight Function:syntax: refine X
776 relaunch the computation of the weight which have a precision lower than X"""
777 args = self.split_arg(line)
778 self.check_refine(args)
779 self.configure()
780
781 self.clean_old_run(keep_event=True)
782 if not self.force:
783 cards = ['madweight_card.dat']
784 self.ask_edit_cards(cards, mode='fixed', plot=False)
785 self.exec_cmd("treatcards")
786 self.do_compile('', refine=True)
787
788 nb_events_by_file = self.MWparam['mw_run']['nb_event_by_node'] * self.MWparam['mw_run']['event_packing']
789 asked_events = self.MWparam['mw_run']['nb_exp_events']
790
791 precision = args[0]
792 name = self.MWparam.name
793 allow_refine = []
794
795 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'r')
796 for line in fsock:
797 if '#' in line:
798 line = line.split('#')[0]
799 line = line.split()
800 if len(line) == 5:
801 lhco_nb, card_nb, tf_set, value, error = line
802 else:
803 continue
804 if tf_set != '1':
805 continue
806 if float(value) * precision < float(error):
807 allow_refine.append((int(card_nb), lhco_nb))
808 logger.info("%s selected jobs for the refinment." % len(allow_refine))
809 for MWdir in self.MWparam.MW_listdir:
810
811 event_to_file = {}
812 for evt_nb in range(asked_events//nb_events_by_file +1):
813 evt = 'verif_%i.lhco' % evt_nb
814 for line in open(pjoin(self.me_dir, 'SubProcesses', MWdir, name, evt)):
815 split = line.split()
816 if len(split) == 3:
817 event_to_file[split[1]] = evt_nb
818 to_refine = {}
819 out_dir = pjoin(self.me_dir, 'Events',name, MWdir)
820 xml_reader = MWParserXML(keep_level='weight')
821 data = xml_reader.read_file(pjoin(out_dir, 'output.xml'))
822 generator = ((int(i),j,int(k),data[i][j][k]) for i in data
823 for j in data[i]
824 for k in data[i][j])
825 for card, event, tf_set, obj in generator:
826 if tf_set != 1:
827 continue
828 value, error = obj.value, obj.error
829 if value * precision < error:
830 if card not in to_refine:
831 to_refine[card] = []
832 if (card,event) in allow_refine:
833 to_refine[card].append(event)
834 if to_refine:
835 self.resubmit(MWdir, to_refine, event_to_file)
836
837
838 starttime = time.time()
839 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'),
840 starttime=starttime, level='madweight', update_results=False)
841 try:
842 self.cluster.wait(self.me_dir, update_status)
843 except Exception:
844 self.cluster.remove()
845 raise
846 except KeyboardInterrupt:
847 if not self.force:
848 ans = self.ask('Error detected. Do you want to clean the queue?',
849 default = 'y', choices=['y','n'])
850 else:
851 ans = 'y'
852 if ans == 'y':
853 self.cluster.remove()
854 raise
855
856 self.do_collect('-refine')
857
858 - def resubmit(self, M_path, to_refine, event_to_file):
859 """resubmit various jobs"""
860
861 for card, event_list in to_refine.items():
862 packets = {}
863 for event in event_list:
864 evt_nb_file = event_to_file[event]
865 if evt_nb_file in packets:
866 packets[evt_nb_file].append(event)
867 else:
868 packets[evt_nb_file] = [event]
869
870
871
872
873 max_evts = self.MWparam['mw_run']['nb_event_by_node']
874 for evt_nb, evt_list in packets.items():
875 restrict_path = pjoin(self.me_dir, 'SubProcesses', M_path, self.MWparam.name,
876 'restrict%i_%i.dat' % (card,evt_nb_file))
877 open(restrict_path, 'w').write(' '.join(map(str, evt_list)))
878
879 nb_weights = len(evt_list)
880 for i in range(1+ (nb_weights-1)//max_evts):
881
882 self.submit_job(M_path, card, i, evt_file=evt_nb,
883 restrict_evt=True)
884
885
886
887
888
889
891 """The command line processor of MadGraph"""
892 pass
893
894
896 pass
897
898
900
901 for card, CardDATA in other.items():
902 if card not in self:
903 self[card] = other[card]
904 continue
905
906 for event, obj2 in CardDATA.items():
907 self[card][event] = obj2
908
909
910
911 - def write(self, out_path, MWdir):
912 """ """
913 fsock = open(out_path, 'w')
914 fsock.write('<subprocess id=\'%s\'>\n' % MWdir)
915 for card in self:
916 fsock.write('<card id=\'%s\'>\n' % card)
917 for event in self[card].values():
918 event.write(fsock)
919 fsock.write('</card>\n')
920
921 fsock.write('</subprocess>')
922 fsock.close()
923
924
925
927 """ This class will organize in python obect the TF_param.dat file
928 (written in xml)
929 """
930
931
932 - def __init__(self, keep_level='weight'):
933 self.in_el = {'process': '', 'card':'', 'event':'', 'tfset':'' ,
934 'permutation':'', 'channel':'','full':''}
935 if keep_level in ['weight', 'event', 'debug']:
936 keep_level = 'tfset'
937 self.all_event = {}
938 self.keep_level = keep_level
939 self.buffer=''
940 self.output = CollectObj()
941
942
943
945
946
947 if self.in_el[self.keep_level] != '' or name == 'log':
948 return
949
950 obj_class = {'event': MW_driver.Weight, 'permutation':MW_driver.Permutation,
951 'channel': MW_driver.Channel, 'tfset':MW_driver.TFsets}
952
953
954 if name == "process":
955 pass
956 elif name == 'card':
957 id = attributes['id']
958 if id not in self.output:
959 self.output[id] = {}
960 self.in_el[name] = self.output[id]
961 self.curr_card_id = id
962 elif name == 'event':
963 id = attributes['id']
964 value = float(attributes['value'])
965 error = float(attributes['error'])
966 card = self.curr_card_id
967 if ((card,id) in self.all_event):
968 data = self.all_event[(card,id)]
969 else:
970 data = MW_driver.Weight(id, self.keep_level)
971 self.all_event[(card,id)] = data
972 data.value = value
973 data.error = error
974 self.in_el['event'] = data
975
976 card = self.in_el['card']
977 card[id] = data
978 elif name == 'tfset':
979 id = attributes['id']
980 value = float(attributes['value'])
981 error = float(attributes['error'])
982 data = MW_driver.TFsets(id)
983 data.value = value
984 data.error = error
985 data.add('0', '', value, error, '')
986
987 event = self.in_el['event']
988 event[id] = data
989 self.in_el['tfset'] = data
990 elif name == 'permutation':
991 tfset = self.in_el['tfset']
992
993 id = attributes['id']
994 value = float(attributes['value'])
995 error = float(attributes['error'])
996 data = MW_driver.Permutation(id, '')
997 data.value = value
998 data.error = error
999 self.in_el['permutation'] = data
1000
1001 tfset[id] = data
1002 elif name == 'channel':
1003 id = attributes['id']
1004 value = float(attributes['value'])
1005 error = float(attributes['error'])
1006 data = MW_driver.Channel(id, value, error)
1007 self.in_el['channel'] = data
1008
1009 permutation = self.in_el['permutation']
1010 permutation[id] = data
1011 elif name in ['log','subprocess','br']:
1012 pass
1013 else:
1014 raise Exception(name)
1015 if name != 'br':
1016 self.text = StringIO()
1017
1019 self.text.write(content)
1020
1022 if name == 'log':
1023 data = self.in_el['event']
1024 data.log = self.text.getvalue()
1025
1026 self.in_el[name] = ''
1027
1028
1036