diff -Nru maatkit-7486/bin/mk-archiver maatkit-7540/bin/mk-archiver --- maatkit-7486/bin/mk-archiver 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-archiver 2011-06-08 16:49:31.000000000 +0000 @@ -24,8 +24,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.27'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7460 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -2716,7 +2716,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -2990,35 +2990,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -3026,8 +3026,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -3069,7 +3069,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -3085,12 +3085,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -5281,8 +5281,7 @@ Calculate L<"--sleep"> as a multiple of the last SELECT time. If this option is specified, mk-archiver will sleep for the query time of the -last SELECT multiplied by the specified coefficient. This option is ignored -if L<"--sleep"> is specified. +last SELECT multiplied by the specified coefficient. This is a slightly more sophisticated way to throttle the SELECTs: sleep a varying amount of time between each SELECT, depending on how long the SELECTs @@ -5800,6 +5799,6 @@ =head1 VERSION -This manual page documents Ver 1.0.27 Distrib 7486 $Revision: 7460 $. +This manual page documents Ver 1.0.27 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-checksum-filter maatkit-7540/bin/mk-checksum-filter --- maatkit-7486/bin/mk-checksum-filter 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-checksum-filter 2011-06-08 16:49:31.000000000 +0000 @@ -23,8 +23,8 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '1.2.22'; -our $DISTRIB = '7486'; +our $VERSION = '1.2.23'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -1392,6 +1392,6 @@ =head1 VERSION -This manual page documents Ver 1.2.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.2.23 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-config-diff maatkit-7540/bin/mk-config-diff --- maatkit-7486/bin/mk-config-diff 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-config-diff 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.0'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -3261,6 +3261,6 @@ =head1 VERSION -This manual page documents Ver 1.0.0 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.0 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-deadlock-logger maatkit-7540/bin/mk-deadlock-logger --- maatkit-7486/bin/mk-deadlock-logger 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-deadlock-logger 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.21'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -2736,6 +2736,6 @@ =head1 VERSION -This manual page documents Ver 1.0.21 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.21 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-duplicate-key-checker maatkit-7540/bin/mk-duplicate-key-checker --- maatkit-7486/bin/mk-duplicate-key-checker 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-duplicate-key-checker 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.2.15'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -4258,6 +4258,6 @@ =head1 VERSION -This manual page documents Ver 1.2.15 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.2.15 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-error-log maatkit-7540/bin/mk-error-log --- maatkit-7486/bin/mk-error-log 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-error-log 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.3'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -2841,6 +2841,6 @@ =head1 VERSION -This manual page documents Ver 1.0.3 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.3 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-fifo-split maatkit-7540/bin/mk-fifo-split --- maatkit-7486/bin/mk-fifo-split 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-fifo-split 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.7'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -1537,6 +1537,6 @@ =head1 VERSION -This manual page documents Ver 1.0.7 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.7 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-find maatkit-7540/bin/mk-find --- maatkit-7486/bin/mk-find 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-find 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.23'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -3812,6 +3812,6 @@ =head1 VERSION -This manual page documents Ver 0.9.23 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.23 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-heartbeat maatkit-7540/bin/mk-heartbeat --- maatkit-7486/bin/mk-heartbeat 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-heartbeat 2011-06-08 16:49:31.000000000 +0000 @@ -1,6 +1,6 @@ #!/usr/bin/env perl -# This is mk-heartbeat, a script to measure replication delay. +# This is mk-heartbeat, a tool to measure replication delay. # # This program is copyright 2007-2011 Percona Inc. # and copyright 2006 Proven Scaling LLC and Six Apart Ltd. @@ -23,12 +23,12 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '1.0.22'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $VERSION = '1.0.23'; +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7537 $ =~ m/(\d+)/g, 0)); # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -302,35 +302,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -338,8 +338,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -381,7 +381,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -397,12 +397,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -2534,6 +2534,657 @@ # ########################################################################### # ########################################################################### +# TableParser package 7156 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/TableParser.pm +# trunk/common/t/TableParser.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### + +package TableParser; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +sub new { + my ( $class, %args ) = @_; + my @required_args = qw(Quoter); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my $self = { %args }; + return bless $self, $class; +} + +sub parse { + my ( $self, $ddl, $opts ) = @_; + return unless $ddl; + if ( ref $ddl eq 'ARRAY' ) { + if ( lc $ddl->[0] eq 'table' ) { + $ddl = $ddl->[1]; + } + else { + return { + engine => 'VIEW', + }; + } + } + + if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) { + die "Cannot parse table definition; is ANSI quoting " + . "enabled or SQL_QUOTE_SHOW_CREATE disabled?"; + } + + my ($name) = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/; + (undef, $name) = $self->{Quoter}->split_unquote($name) if $name; + + $ddl =~ s/(`[^`]+`)/\L$1/g; + + my $engine = $self->get_engine($ddl); + + my @defs = $ddl =~ m/^(\s+`.*?),?$/gm; + my @cols = map { $_ =~ m/`([^`]+)`/ } @defs; + MKDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols)); + + my %def_for; + @def_for{@cols} = @defs; + + my (@nums, @null); + my (%type_for, %is_nullable, %is_numeric, %is_autoinc); + foreach my $col ( @cols ) { + my $def = $def_for{$col}; + my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/; + die "Can't determine column type for $def" unless $type; + $type_for{$col} = $type; + if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) { + push @nums, $col; + $is_numeric{$col} = 1; + } + if ( $def !~ m/NOT NULL/ ) { + push @null, $col; + $is_nullable{$col} = 1; + } + $is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0; + } + + my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable); + + my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/; + + return { + name => $name, + cols => \@cols, + col_posn => { map { $cols[$_] => $_ } 0..$#cols }, + is_col => { map { $_ => 1 } @cols }, + null_cols => \@null, + is_nullable => \%is_nullable, + is_autoinc => \%is_autoinc, + clustered_key => $clustered_key, + keys => $keys, + defs => \%def_for, + numeric_cols => \@nums, + is_numeric => \%is_numeric, + engine => $engine, + type_for => \%type_for, + charset => $charset, + }; +} + +sub sort_indexes { + my ( $self, $tbl ) = @_; + + my @indexes + = sort { + (($a ne 'PRIMARY') <=> ($b ne 'PRIMARY')) + || ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} ) + || ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} ) + || ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) ) + } + grep { + $tbl->{keys}->{$_}->{type} eq 'BTREE' + } + sort keys %{$tbl->{keys}}; + + MKDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes)); + return @indexes; +} + +sub find_best_index { + my ( $self, $tbl, $index ) = @_; + my $best; + if ( $index ) { + ($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}}; + } + if ( !$best ) { + if ( $index ) { + die "Index '$index' does not exist in table"; + } + else { + ($best) = $self->sort_indexes($tbl); + } + } + MKDEBUG && _d('Best index found is', $best); + return $best; +} + +sub find_possible_keys { + my ( $self, $dbh, $database, $table, $quoter, $where ) = @_; + return () unless $where; + my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table) + . ' WHERE ' . $where; + MKDEBUG && _d($sql); + my $expl = $dbh->selectrow_hashref($sql); + $expl = { map { lc($_) => $expl->{$_} } keys %$expl }; + if ( $expl->{possible_keys} ) { + MKDEBUG && _d('possible_keys =', $expl->{possible_keys}); + my @candidates = split(',', $expl->{possible_keys}); + my %possible = map { $_ => 1 } @candidates; + if ( $expl->{key} ) { + MKDEBUG && _d('MySQL chose', $expl->{key}); + unshift @candidates, grep { $possible{$_} } split(',', $expl->{key}); + MKDEBUG && _d('Before deduping:', join(', ', @candidates)); + my %seen; + @candidates = grep { !$seen{$_}++ } @candidates; + } + MKDEBUG && _d('Final list:', join(', ', @candidates)); + return @candidates; + } + else { + MKDEBUG && _d('No keys in possible_keys'); + return (); + } +} + +sub check_table { + my ( $self, %args ) = @_; + my @required_args = qw(dbh db tbl); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($dbh, $db, $tbl) = @args{@required_args}; + my $q = $self->{Quoter}; + my $db_tbl = $q->quote($db, $tbl); + MKDEBUG && _d('Checking', $db_tbl); + + my $sql = "SHOW TABLES FROM " . $q->quote($db) + . ' LIKE ' . $q->literal_like($tbl); + MKDEBUG && _d($sql); + my $row; + eval { + $row = $dbh->selectrow_arrayref($sql); + }; + if ( $EVAL_ERROR ) { + MKDEBUG && _d($EVAL_ERROR); + return 0; + } + if ( !$row->[0] || $row->[0] ne $tbl ) { + MKDEBUG && _d('Table does not exist'); + return 0; + } + + MKDEBUG && _d('Table exists; no privs to check'); + return 1 unless $args{all_privs}; + + $sql = "SHOW FULL COLUMNS FROM $db_tbl"; + MKDEBUG && _d($sql); + eval { + $row = $dbh->selectrow_hashref($sql); + }; + if ( $EVAL_ERROR ) { + MKDEBUG && _d($EVAL_ERROR); + return 0; + } + if ( !scalar keys %$row ) { + MKDEBUG && _d('Table has no columns:', Dumper($row)); + return 0; + } + my $privs = $row->{privileges} || $row->{Privileges}; + + $sql = "DELETE FROM $db_tbl LIMIT 0"; + MKDEBUG && _d($sql); + eval { + $dbh->do($sql); + }; + my $can_delete = $EVAL_ERROR ? 0 : 1; + + MKDEBUG && _d('User privs on', $db_tbl, ':', $privs, + ($can_delete ? 'delete' : '')); + + if ( !($privs =~ m/select/ && $privs =~ m/insert/ && $privs =~ m/update/ + && $can_delete) ) { + MKDEBUG && _d('User does not have all privs'); + return 0; + } + + MKDEBUG && _d('User has all privs'); + return 1; +} + +sub get_engine { + my ( $self, $ddl, $opts ) = @_; + my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; + MKDEBUG && _d('Storage engine:', $engine); + return $engine || undef; +} + +sub get_keys { + my ( $self, $ddl, $opts, $is_nullable ) = @_; + my $engine = $self->get_engine($ddl); + my $keys = {}; + my $clustered_key = undef; + + KEY: + foreach my $key ( $ddl =~ m/^ ((?:[A-Z]+ )?KEY .*)$/gm ) { + + next KEY if $key =~ m/FOREIGN/; + + my $key_ddl = $key; + MKDEBUG && _d('Parsed key:', $key_ddl); + + if ( $engine !~ m/MEMORY|HEAP/ ) { + $key =~ s/USING HASH/USING BTREE/; + } + + my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/; + my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/; + $type = $type || $special || 'BTREE'; + if ( $opts->{mysql_version} && $opts->{mysql_version} lt '004001000' + && $engine =~ m/HEAP|MEMORY/i ) + { + $type = 'HASH'; # MySQL pre-4.1 supports only HASH indexes on HEAP + } + + my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/; + my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0; + my @cols; + my @col_prefixes; + foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) { + my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/; + push @cols, $name; + push @col_prefixes, $prefix; + } + $name =~ s/`//g; + + MKDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols)); + + $keys->{$name} = { + name => $name, + type => $type, + colnames => $cols, + cols => \@cols, + col_prefixes => \@col_prefixes, + is_unique => $unique, + is_nullable => scalar(grep { $is_nullable->{$_} } @cols), + is_col => { map { $_ => 1 } @cols }, + ddl => $key_ddl, + }; + + if ( $engine =~ m/InnoDB/i && !$clustered_key ) { + my $this_key = $keys->{$name}; + if ( $this_key->{name} eq 'PRIMARY' ) { + $clustered_key = 'PRIMARY'; + } + elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) { + $clustered_key = $this_key->{name}; + } + MKDEBUG && $clustered_key && _d('This key is the clustered key'); + } + } + + return $keys, $clustered_key; +} + +sub get_fks { + my ( $self, $ddl, $opts ) = @_; + my $fks = {}; + + foreach my $fk ( + $ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg ) + { + my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/; + my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/; + my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/; + + if ( $parent !~ m/\./ && $opts->{database} ) { + $parent = "`$opts->{database}`.$parent"; + } + + $fks->{$name} = { + name => $name, + colnames => $cols, + cols => [ map { s/[ `]+//g; $_; } split(',', $cols) ], + parent_tbl => $parent, + parent_colnames=> $parent_cols, + parent_cols => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ], + ddl => $fk, + }; + } + + return $fks; +} + +sub remove_auto_increment { + my ( $self, $ddl ) = @_; + $ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m; + return $ddl; +} + +sub remove_secondary_indexes { + my ( $self, $ddl ) = @_; + my $sec_indexes_ddl; + my $tbl_struct = $self->parse($ddl); + + if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) { + my $clustered_key = $tbl_struct->{clustered_key}; + $clustered_key ||= ''; + + my @sec_indexes = map { + my $key_def = $_->{ddl}; + $key_def =~ s/([\(\)])/\\$1/g; + $ddl =~ s/\s+$key_def//i; + + my $key_ddl = "ADD $_->{ddl}"; + $key_ddl .= ',' unless $key_ddl =~ m/,$/; + $key_ddl; + } + grep { $_->{name} ne $clustered_key } + values %{$tbl_struct->{keys}}; + MKDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes)); + + if ( @sec_indexes ) { + $sec_indexes_ddl = join(' ', @sec_indexes); + $sec_indexes_ddl =~ s/,$//; + } + + $ddl =~ s/,(\n\) )/$1/s; + } + else { + MKDEBUG && _d('Not removing secondary indexes from', + $tbl_struct->{engine}, 'table'); + } + + return $ddl, $sec_indexes_ddl, $tbl_struct; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; + +# ########################################################################### +# End TableParser package +# ########################################################################### + +# ########################################################################### +# Transformers package 7226 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/Transformers.pm +# trunk/common/t/Transformers.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### + +package Transformers; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Time::Local qw(timegm timelocal); +use Digest::MD5 qw(md5_hex); + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +require Exporter; +our @ISA = qw(Exporter); +our %EXPORT_TAGS = (); +our @EXPORT = (); +our @EXPORT_OK = qw( + micro_t + percentage_of + secs_to_time + time_to_secs + shorten + ts + parse_timestamp + unix_timestamp + any_unix_timestamp + make_checksum + crc32 +); + +our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/; +our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/; +our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks + +sub micro_t { + my ( $t, %args ) = @_; + my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals + my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals + my $f; + + $t = 0 if $t < 0; + + $t = sprintf('%.17f', $t) if $t =~ /e/; + + $t =~ s/\.(\d{1,6})\d*/\.$1/; + + if ($t > 0 && $t <= 0.000999) { + $f = ($t * 1000000) . 'us'; + } + elsif ($t >= 0.001000 && $t <= 0.999999) { + $f = sprintf("%.${p_ms}f", $t * 1000); + $f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros + } + elsif ($t >= 1) { + $f = sprintf("%.${p_s}f", $t); + $f = ($f * 1) . 's'; # * 1 to remove insignificant zeros + } + else { + $f = 0; # $t should = 0 at this point + } + + return $f; +} + +sub percentage_of { + my ( $is, $of, %args ) = @_; + my $p = $args{p} || 0; # float precision + my $fmt = $p ? "%.${p}f" : "%d"; + return sprintf $fmt, ($is * 100) / ($of ||= 1); +} + +sub secs_to_time { + my ( $secs, $fmt ) = @_; + $secs ||= 0; + return '00:00' unless $secs; + + $fmt ||= $secs >= 86_400 ? 'd' + : $secs >= 3_600 ? 'h' + : 'm'; + + return + $fmt eq 'd' ? sprintf( + "%d+%02d:%02d:%02d", + int($secs / 86_400), + int(($secs % 86_400) / 3_600), + int(($secs % 3_600) / 60), + $secs % 60) + : $fmt eq 'h' ? sprintf( + "%02d:%02d:%02d", + int(($secs % 86_400) / 3_600), + int(($secs % 3_600) / 60), + $secs % 60) + : sprintf( + "%02d:%02d", + int(($secs % 3_600) / 60), + $secs % 60); +} + +sub time_to_secs { + my ( $val, $default_suffix ) = @_; + die "I need a val argument" unless defined $val; + my $t = 0; + my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; + $suffix = $suffix || $default_suffix || 's'; + if ( $suffix =~ m/[smhd]/ ) { + $t = $suffix eq 's' ? $num * 1 # Seconds + : $suffix eq 'm' ? $num * 60 # Minutes + : $suffix eq 'h' ? $num * 3600 # Hours + : $num * 86400; # Days + + $t *= -1 if $prefix && $prefix eq '-'; + } + else { + die "Invalid suffix for $val: $suffix"; + } + return $t; +} + +sub shorten { + my ( $num, %args ) = @_; + my $p = defined $args{p} ? $args{p} : 2; # float precision + my $d = defined $args{d} ? $args{d} : 1_024; # divisor + my $n = 0; + my @units = ('', qw(k M G T P E Z Y)); + while ( $num >= $d && $n < @units - 1 ) { + $num /= $d; + ++$n; + } + return sprintf( + $num =~ m/\./ || $n + ? "%.${p}f%s" + : '%d', + $num, $units[$n]); +} + +sub ts { + my ( $time, $gmt ) = @_; + my ( $sec, $min, $hour, $mday, $mon, $year ) + = $gmt ? gmtime($time) : localtime($time); + $mon += 1; + $year += 1900; + my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d", + $year, $mon, $mday, $hour, $min, $sec); + if ( my ($us) = $time =~ m/(\.\d+)$/ ) { + $us = sprintf("%.6f", $us); + $us =~ s/^0\././; + $val .= $us; + } + return $val; +} + +sub parse_timestamp { + my ( $val ) = @_; + if ( my($y, $m, $d, $h, $i, $s, $f) + = $val =~ m/^$mysql_ts$/ ) + { + return sprintf "%d-%02d-%02d %02d:%02d:" + . (defined $f ? '%09.6f' : '%02d'), + $y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s); + } + return $val; +} + +sub unix_timestamp { + my ( $val, $gmt ) = @_; + if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) { + $val = $gmt + ? timegm($s, $i, $h, $d, $m - 1, $y) + : timelocal($s, $i, $h, $d, $m - 1, $y); + if ( defined $us ) { + $us = sprintf('%.6f', $us); + $us =~ s/^0\././; + $val .= $us; + } + } + return $val; +} + +sub any_unix_timestamp { + my ( $val, $callback ) = @_; + + if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) { + $n = $suffix eq 's' ? $n # Seconds + : $suffix eq 'm' ? $n * 60 # Minutes + : $suffix eq 'h' ? $n * 3600 # Hours + : $suffix eq 'd' ? $n * 86400 # Days + : $n; # default: Seconds + MKDEBUG && _d('ts is now - N[shmd]:', $n); + return time - $n; + } + elsif ( $val =~ m/^\d{9,}/ ) { + MKDEBUG && _d('ts is already a unix timestamp'); + return $val; + } + elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) { + MKDEBUG && _d('ts is MySQL slow log timestamp'); + $val .= ' 00:00:00' unless $hms; + return unix_timestamp(parse_timestamp($val)); + } + elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) { + MKDEBUG && _d('ts is properly formatted timestamp'); + $val .= ' 00:00:00' unless $hms; + return unix_timestamp($val); + } + else { + MKDEBUG && _d('ts is MySQL expression'); + return $callback->($val) if $callback && ref $callback eq 'CODE'; + } + + MKDEBUG && _d('Unknown ts type:', $val); + return; +} + +sub make_checksum { + my ( $val ) = @_; + my $checksum = uc substr(md5_hex($val), -16); + MKDEBUG && _d($checksum, 'checksum for', $val); + return $checksum; +} + +sub crc32 { + my ( $string ) = @_; + return unless $string; + my $poly = 0xEDB88320; + my $crc = 0xFFFFFFFF; + foreach my $char ( split(//, $string) ) { + my $comp = ($crc ^ ord($char)) & 0xFF; + for ( 1 .. 8 ) { + $comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1; + } + $crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp; + } + return $crc ^ 0xFFFFFFFF; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; + +# ########################################################################### +# End Transformers package +# ########################################################################### + +# ########################################################################### # This is a combination of modules and programs in one -- a runnable module. # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. @@ -2545,17 +3196,23 @@ use English qw(-no_match_vars); use List::Util qw(min max sum); -use Time::HiRes qw(usleep gettimeofday); +use Time::HiRes qw(gettimeofday time sleep usleep); use IO::File; use constant MKDEBUG => $ENV{MKDEBUG} || 0; +Transformers->import qw(ts unix_timestamp); + my @dbhs; # Holds slave DBHs if --recurse my @sths; # Holds [$host, $sth] if --recurse sub main { @ARGV = @_; # set global ARGV for this package + # Reset all global vars between test runs else weird things happen. + @dbhs = (); + @sths = (); + # ######################################################################## # Get configuration information. # ######################################################################## @@ -2563,8 +3220,11 @@ $o->get_specs(); $o->get_opts(); - if ( !$o->get('help') ) { + my $dp = $o->DSNParser; + $dp->prop('dbidriver', $o->get('dbi-driver')); + $dp->prop('set-vars', $o->get('set-vars')); + if ( !$o->get('help') ) { my @frames = $o->get('frames') =~ m/(\d+[smhd])/g; if ( @frames ) { my @times; @@ -2591,20 +3251,33 @@ && !($o->get('database') && $o->get('table'))) { $o->save_error('--create-table requires both --database and --table'); } - } - $o->set('interval', max($o->get('interval'), 1)); # Prevent 0-second interval + if ( $o->get('interval') < 0.01 ) { + $o->save_error("--interval must be >= 0.01"); + } + + if ( !$o->get('stop') && !$o->get('database') ) { + $o->save_error('--database must be specified'); + } + } $o->usage_or_errors(); - # Frequently used options. + # ######################################################################## + # Make common modules and var for frequently used options. + # ######################################################################## + my $q = new Quoter(); + my $tp = new TableParser(Quoter => $q); + my $interval = $o->get('interval'); + my $skew = $o->get('update') ? 0 : $o->get('skew'); my $sentinel = $o->get('sentinel'); - my $table = $o->get('table'); my $frames = $o->get('frames'); + my $db = $o->get('database'); + my $tbl = $o->get('table'); # ######################################################################## - # First things first: if --stop was given, create the sentinel file. + # Create --sentinel file if --stop was given, and possibly exit. # ######################################################################## if ( $o->get('stop') ) { MKDEBUG && _d('Creating sentinel file', $sentinel); @@ -2614,8 +3287,7 @@ or die "Cannot write to $sentinel: $OS_ERROR\n"; close $file or die "Cannot close $sentinel: $OS_ERROR\n"; - print STDOUT "Successfully created file $sentinel\n" - unless $o->get('quiet'); + print STDOUT "Successfully created file $sentinel\n"; # Exit only if no other action (update, monitor, check) is given. if ( !$o->get('update') && !$o->get('check') && !$o->get('monitor') ) { MKDEBUG && _d("Nothing more to do, quitting"); @@ -2626,7 +3298,7 @@ # same --interval as this invocation. Then remove the file and # continue. MKDEBUG && _d("Waiting for other instances to quit"); - sleep($interval); + sleep $interval ; MKDEBUG && _d("Unlinking", $sentinel); unlink $sentinel or die "Cannot unlink $sentinel: $OS_ERROR"; @@ -2634,67 +3306,268 @@ } # ######################################################################## - # Work. + # Connect to MySQL. # ######################################################################## - my $update_sql = $o->get('replace') - ? "REPLACE INTO $table(id, ts) VALUES(1, NOW())" - : "UPDATE $table SET ts = NOW() WHERE id = 1"; - my $select_sql = "SELECT " - . ( $o->get('dbi-driver') eq 'Pg' - ? "ROUND(DATE_PART('epoch', NOW() - ts)) AS delay" - : 'UNIX_TIMESTAMP() - UNIX_TIMESTAMP(ts) AS delay/*!50038, @@hostname AS host*/' ) - . " FROM $table WHERE id = 1"; - - MKDEBUG && _d("UPDATE SQL:", $update_sql); - MKDEBUG && _d("SELECT SQL:", $select_sql); - - # Get the password before daemonizing. if ( $o->get('ask-pass') ) { $o->set('password', OptionParser::prompt_noecho("Enter password: ")); } - - my $dp = $o->DSNParser; - $dp->prop('dbidriver', $o->get('dbi-driver')); - $dp->prop('set-vars', $o->get('set-vars')); my $dsn_defaults = $dp->parse_options($o); my $dsn = @ARGV ? $dp->parse(shift @ARGV, $dsn_defaults) : $dsn_defaults; - my $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), - { AutoCommit => 1, }); - my $sth = $dbh->prepare($o->get('update') ? $update_sql : $select_sql); + my $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), {AutoCommit=>1}); $dbh->{InactiveDestroy} = 1; # Don't disconnect on fork + $dbh->{FetchHashKeyName} = 'NAME_lc'; + $dbh->do("USE `$db`"); - # Create the heartbeat table if desired. + # ######################################################################## + # Create the heartbeat table if --create-table was given. + # ######################################################################## + my $db_tbl = $q->quote($db, $tbl); + my $server_id = $dbh->selectrow_array('SELECT @@server_id'); if ( $o->get('create-table') ) { - my $q = new Quoter(); - my $db_tbl = $q->quote($o->get('database'), $o->get('table')); - my $sql = $o->read_para_after( - __FILE__, qr/MAGIC_create_heartbeat/); + my $sql = $o->read_para_after(__FILE__, qr/MAGIC_create_heartbeat/); $sql =~ s/heartbeat/IF NOT EXISTS $db_tbl/; MKDEBUG && _d($sql); $dbh->do($sql); - $sql = "INSERT INTO $db_tbl (id) VALUES (1)"; + $sql = "INSERT INTO $db_tbl (ts, server_id) VALUES (NOW(), $server_id)"; MKDEBUG && _d($sql); # This may fail if the table already existed and already had this row. # We eval to ignore this possibility. eval { $dbh->do($sql); }; } + # ######################################################################## + # Get and check heartbeat table structure. + # ######################################################################## + my $tbl_def = $dbh->selectrow_arrayref("SHOW CREATE TABLE $db_tbl"); + my $tbl_struct = $tp->parse($tbl_def->[1]); + + die "Heartbeat table $db_tbl does not have a ts column" + unless $tbl_struct->{is_col}->{ts}; + + my $hires_ts = $tbl_struct->{type_for}->{ts} =~ m/char/i ? 1 : 0; + MKDEBUG && _d("Hi-res ts:", ($hires_ts ? 'yes' : 'no')); + + my $id = $tbl_struct->{is_col}->{id}; # legacy table struct + die "Heartbeat table $db_tbl does not have a server_id or id column" + unless $tbl_struct->{is_col}->{server_id} || $id; + + # If there's an id column, then we're running in legacy mode. If there's + # a server_id column, then we're running in the new mode which supports + # multiple --update instances. + if ( $tbl_struct->{is_col}->{id} && $tbl_struct->{is_col}->{server_id} ) { + die "Heartbeat table $db_tbl cannot have both an id column and " + . "a server_id column"; + } + + # pk_col and pk_val are used to identify the heartbeat row to update or + # or monitor. + my ($pk_col, $pk_val); + if ( $id ) { + # Legacy mode: update heartbeat row WHERE id=1 and monitor heartbeat + # row WHERE id=1. + $pk_col = 'id'; + $pk_val = '1'; + } + elsif ( $tbl_struct->{is_col}->{server_id} ) { + # Multi-update mode: update heartbeat row WHERE server_id=@@server_id + # and monitor heartbeat row WHERE server_id=master_server_id. + if ( $o->get('update') ) { + $pk_col = 'server_id'; + $pk_val = $server_id; + } + else { # monitor or check + my $master_server_id = $o->get('master-server-id'); + if ( !$master_server_id ) { + eval { + my $vp = new VersionParser(); + my $ms = new MasterSlave(VersionParser => $vp); + my $master_dsn = $ms->get_master_dsn($dbh, $dsn, $dp) + or die "This server is not a slave"; + my $master_dbh = $dp->get_dbh($dp->get_cxn_params($master_dsn), + { AutoCommit => 1 }); + ($master_server_id) + = $master_dbh->selectrow_array('SELECT @@server_id'); + $master_dbh->disconnect; + }; + if ( $EVAL_ERROR ) { + MKDEBUG && _d("Error determining master id:", $EVAL_ERROR); + } + } + if ( !$master_server_id ) { + die "The --master-server-id option must be specified because " + . "the heartbeat table $db_tbl uses the server_id column " + . "for --update or --check but the server's master could " + . "not be automatically determined.\n" + . "Please read the DESCRIPTION section of the mk-heartbeat POD.\n"; + } + $pk_col = 'server_id'; + $pk_val = $master_server_id; + } + } + else { + die "Heartbeat table $db_tbl does not have a server_id or id column"; + } + MKDEBUG && _d('Heartbeat row primary key:', $pk_col, '=', $pk_val); + + # Check that heartbeat table has at least 1 row unless --replace because + # --replace will create the row if it doesn't exist. + if ( !$o->get('replace') ) { + my $sql = "SELECT 1 FROM $db_tbl WHERE $pk_col='$pk_val' LIMIT 1"; + MKDEBUG && _d($sql); + my $row = $dbh->selectall_arrayref($sql); + if ( scalar @$row == 0 ) { + MKDEBUG && _d('No heartbeat row in table'); + if ( $o->get('insert-heartbeat-row') ) { + my $sql = "INSERT INTO $db_tbl ($pk_col, ts) " + . "VALUES ('$pk_val', NOW())"; + MKDEBUG && _d($sql); + $dbh->do($sql); + } + else { + if ( $id ) { + die "The heartbeat table is empty.\n" + . "At least one row must be inserted into the heartbeat " + . "table.\nPlease read the DESCRIPTION section of the " + . "mk-heartbeat POD.\n"; + } + else { + die "No row found in heartbeat table for server_id $pk_val.\n" + . "At least one row must be inserted into the heartbeat " + . "table for server_id $pk_val.\nPlease read the " + . "DESCRIPTION section of the mk-heartbeat POD.\n"; + } + } + } + } + + # ######################################################################## + # Make sth for updating or checking the heartbeat table. + # ######################################################################## + my ($heartbeat_sql, $heartbeat_sth); + my ($get_delay, $update_heartbeat); + + if ( $o->get('update') ) { + my @master_status_cols = grep { $tbl_struct->{is_col}->{$_} } + qw(file position); + MKDEBUG && _d("Master status columns:", join(', ', @master_status_cols)); + + my @slave_status_cols = grep { $tbl_struct->{is_col}->{$_} } + qw(relay_master_log_file exec_master_log_pos); + MKDEBUG && _d("Slave status columns:", join(', ', @slave_status_cols)); + + # Just a shortcut so I don't have to check both arrays when creating + # SQL statement below. + my @extra_cols = (@master_status_cols, @slave_status_cols); + + if ( $o->get('replace') ) { + $heartbeat_sql + = "REPLACE INTO $db_tbl (ts, $pk_col" + . (@extra_cols ? ", " . join(', ', @extra_cols) : '') + . ") VALUES (?, '$pk_val'" + . (@extra_cols ? ", " . join(', ', map { '?' } @extra_cols) : '') + . ")"; + } + else { + $heartbeat_sql + = "UPDATE $db_tbl SET ts=?" + . (@extra_cols ? ", " . join(', ', map { "$_=?" } @extra_cols) : "") + . " WHERE $pk_col='$pk_val'"; + } + MKDEBUG && _d("UPDATE SQL:", $heartbeat_sql); + + $heartbeat_sth = $dbh->prepare($heartbeat_sql); + + $update_heartbeat = sub { + my ($sth) = @_; + my @vals; + + my $sql; + if ( @master_status_cols ) { + $sql = "SHOW MASTER STATUS"; + MKDEBUG && _d($dbh, $sql); + my $row = $dbh->selectrow_hashref($sql); + if ( !$row ) { + MKDEBUG && _d("No row from", $sql); + push @vals, map { undef } @master_status_cols; + } + else { + push @vals, map { $row->{$_} } @master_status_cols; + } + } + + if ( @slave_status_cols ) { + $sql = "SHOW SLAVE STATUS"; + MKDEBUG && _d($dbh, $sql); + my $row = $dbh->selectrow_hashref($sql); + if ( !$row ) { + MKDEBUG && _d("No row from", $sql); + push @vals, map { undef } @slave_status_cols; + } + else { + push @vals, map { $row->{$_} } @slave_status_cols; + } + } + + $sth->execute(ts(time), @vals); + MKDEBUG && _d($sth->{Statement}); + $sth->finish(); + + return; + }; + } + else { # --monitor or --check + my $dbi_driver = lc $o->get('dbi-driver'); + + $heartbeat_sql + = "SELECT ts" + . ($dbi_driver eq 'mysql' ? '/*!50038, @@hostname AS host*/' : '') + . ($id ? "" : ", server_id") + . " FROM $db_tbl " + . "WHERE $pk_col='$pk_val' " + . "LIMIT 1"; + MKDEBUG && _d("SELECT SQL:", $heartbeat_sql); + + $heartbeat_sth = $dbh->prepare($heartbeat_sql); + + $get_delay = sub { + my ($sth) = @_; + $sth->execute(); + MKDEBUG && _d($sth->{Statement}); + my ($ts, $hostname, $server_id) = $sth->fetchrow_array(); + my $now = time; + MKDEBUG && _d("Heartbeat from server", $server_id, "\n", + " now:", ts($now), "\n", + " ts:", $ts, "\n", + "skew:", $skew); + my $delay = $now - unix_timestamp($ts) - $skew; + MKDEBUG && _d('Delay', sprintf('%.6f', $delay), 'on', $hostname); + + # Because we adjust for skew, if the ts are less than skew seconds + # apart (i.e. replication is very fast) then delay will be negative. + # So it's effectively 0 seconds of lag. + $delay = 0.00 if $delay < 0; + + $sth->finish(); + return ($delay, $hostname, $pk_val); + }; + } + # Do a little check just to make sure the table is there, so there's one last # chance to catch errors before daemonizing. - $sth->execute(); - $sth->finish(); - - # Check that heartbeat table has at least 1 row. - my $row = $dbh->selectall_arrayref("SELECT id FROM $table LIMIT 1"); - if ( scalar @$row == 0 ) { - die "The heartbeat table is empty.\n" - . "At least one row must be inserted into the heartbeat table.\n" - . "Please read the DESCRIPTION section of the mk-heartbeat POD.\n"; + if ( $o->get('update') ) { + $update_heartbeat->($heartbeat_sth); + } + else { + $get_delay->($heartbeat_sth); } + $heartbeat_sth->finish(); + # ######################################################################## + # Daemonize only after (potentially) asking for passwords for --ask-pass. + # ######################################################################## my $daemon; if ( $o->get('daemonize') ) { $daemon = new Daemon(o=>$o); @@ -2707,55 +3580,76 @@ $daemon->make_PID_file(); } - # Setup for moving averages. - my @samples; - my $limit = max(@$frames); - my $format = "%4ds [ " . join(", ", map { "%5.2fs" } @$frames) . " ]\n"; - - my $now = time(); - my $end = $now + ($o->get('run-time') || 0); # When to exit - MKDEBUG && _d('Will exit at', $end); - - # --monitor and --update run in a loop, but --check only checks once and - # exits. + # ######################################################################## + # --check and exit if --check was given. + # ######################################################################## if ( $o->get('check') ) { + MKDEBUG && _d('--check and exit'); check_delay( - o => $o, - dp => $dp, - dsn => $dsn, - dbh => $dbh, - sth => $sth, - select_sql => $select_sql, + dsn => $dsn, + dbh => $dbh, + sth => $heartbeat_sth, + sql => $heartbeat_sql, + get_delay => $get_delay, + interval => $interval, + skew => $skew, + hires_ts => $hires_ts, + OptionParser => $o, + DSNParser => $dp, ); - disconnect($dbh, $sth); + disconnect($dbh, $heartbeat_sth); return 0; } - # Quit if time is exceeded or sentinel exists. - while ( (!$o->get('run-time') || $now < $end) && !-f $sentinel ) { - eval { + # ######################################################################## + # Setup moving averages for --frames. + # ######################################################################## + my @samples; + my $limit = max(@$frames); + + # 2.00s [ 0.05s, 0.01s, 0.00s ] + my $format = ($hires_ts ? '%.2f' : '%4d') . "s " + . "[ " . join(", ", map { "%5.2fs" } @$frames) . " ]" + . ($o->get('print-master-server-id') ? " %d" : '') + . "\n"; + + # ######################################################################## + # Monitor or update the heartbeat table. + # ######################################################################## + my $end = $o->get('run-time') ? int(time + $o->get('run-time')) : 0; + MKDEBUG && _d($end ? ('Will exit at', ts($end)) : 'Running forever'); + + my $get_next_interval = make_interval_iter($interval, $skew); - my @now = gettimeofday(); - usleep( - (($interval - ($now[0] % $interval)) * 1_000_000) - $now[1] - # --update happens at the second boundary, but --monitor and --check - # are delayed to avoid spurious results. - + ($o->get('update') ? 0 : $o->get('skew') )); - MKDEBUG && _d('Woke up at', gettimeofday()); + while ( # Stop if... + (!$end || int(time) < $end) # runtime exceeded, or + && !-f $sentinel # sentinel file created + ) { + eval { + my $next_interval = $get_next_interval->(); + if ( time >= $next_interval ) { + do { $next_interval = $get_next_interval->() } + until $next_interval > time; + MKDEBUG && _d("Missed last interval; next interval:", + ts($next_interval)); + } + sleep $next_interval - time; + MKDEBUG && _d('Woke up at', ts(time)); # Connect or reconnect if necessary. if ( !$dbh->ping() ) { $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 }); - $sth = undef; + $dbh->{InactiveDestroy} = 1; # Don't disconnect on fork + $dbh->{FetchHashKeyName} = 'NAME_lc'; + $dbh->do("USE `$db`"); + + $heartbeat_sth = undef; } if ( $o->get('monitor') ) { - # Get the data - $sth ||= $dbh->prepare($select_sql); - $sth->execute(); - MKDEBUG && _d($sth->{Statement}); - my ( $delay ) = $sth->fetchrow_array(); - MKDEBUG && _d('Found delay of', $delay); + $heartbeat_sth ||= $dbh->prepare($heartbeat_sql); + my ($delay) = $get_delay->($heartbeat_sth); + unshift @samples, $delay; pop @samples if @samples > $limit; @@ -2764,7 +3658,8 @@ my $bound = min($_, scalar(@samples)); sum(@samples[0 .. $bound-1]) / $_; } @$frames; - my $output = sprintf($format, $delay, @vals); + + my $output = sprintf $format, $delay, @vals, $pk_val; if ( my $file = $o->get('file') ) { open my $file, '>', $file or die "Can't open $file: $OS_ERROR"; @@ -2774,28 +3669,26 @@ or die "Can't close $file: $OS_ERROR"; } else { - print $output unless $o->get('quiet'); + print $output; } } else { # --update mode - $sth ||= $dbh->prepare($update_sql); - $sth->execute(); - MKDEBUG && _d($sth->{Statement}); + $heartbeat_sth ||= $dbh->prepare($heartbeat_sql); + $update_heartbeat->($heartbeat_sth); } }; if ( $EVAL_ERROR ) { my ( $err ) = $EVAL_ERROR =~ m/^(?:DBI|DBD).*failed: (.*?)\s*at \S+ line .*/; if ( $err ) { - print STDERR $err, "\n"; + warn "$err\n"; } else { die $EVAL_ERROR; } } - $now = time(); } - disconnect($dbh, $sth); + disconnect($dbh, $heartbeat_sth); return 0; } @@ -2806,11 +3699,12 @@ # Check the delay on a single server. Optionally recurse to all its slaves. sub check_delay { my ( %args ) = @_; - foreach my $arg ( qw(dp dsn dbh) ) { + my @required_args = qw(dsn dbh sth sql get_delay interval skew OptionParser DSNParser); + foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my $o = $args{o}; - + my ($dsn, $dbh, $sth, $sql, $get_delay, $interval, $skew, $o, $dp) + = @args{@required_args}; MKDEBUG && _d('Checking slave delay'); # Collect a list of connections to the slaves. @@ -2819,54 +3713,91 @@ my $vp = new VersionParser(); my $ms = new MasterSlave(VersionParser => $vp); $ms->recurse_to_slaves( - { dbh => $args{dbh}, - dsn => $args{dsn}, - dsn_parser => $args{dp}, + { dbh => $dbh, + dsn => $dsn, + dsn_parser => $dp, recurse => $o->get('recurse'), callback => sub { my ( $dsn, $dbh, $level ) = @_; push @dbhs, $dbh; - push @sths, [ $dsn, $dbh->prepare($args{select_sql}) ]; + MKDEBUG && _d("Found slave", $dp->as_string($dsn)); + push @sths, [ $dsn, $dbh->prepare($sql) ]; }, method => $o->get('recursion-method'), }, ); } else { - push @sths, [ $args{dsn}, $args{sth} ]; + push @sths, [ $dsn, $sth ]; } - # Assume that each query will be very fast, so there's no real need to - # sleep between EACH server checked. - my @now = gettimeofday(); - usleep((($o->get('interval') - ($now[0] % $o->get('interval'))) * 1_000_000) - - $now[1] + $o->get('skew')); - MKDEBUG && _d('Woke up at', gettimeofday()); + my $format_delay = ($args{hires_ts} ? '%.2f' : '%d') + . ($o->get('print-master-server-id') ? " %d" : "") + . "\n"; + my $format_host = "%-20s $format_delay"; + # Before hi-res ts, we could check all slaves at one interval, assuming + # the checks were fast, i.e. able to be done within one interval. But + # now we have intervals up to 0.01 fast and that's too short to check all + # slaves. So for each slave we sleep until the next interval. + my $get_next_interval = make_interval_iter($interval, $skew); + + SLAVE: foreach my $thing ( @sths ) { my ( $dsn, $sth ) = @$thing; - my $host = $dsn->{h} || ''; - if ( $dsn->{P} && $dsn->{P} ne '3306' ) { - $host .= ":$dsn->{P}"; - } - $sth->execute(); - my ( $delay, $hostname ) = $sth->fetchrow_array(); - $host ||= $hostname || ''; - MKDEBUG && _d('Found delay of', $delay, 'on', $host); + MKDEBUG && _d('Checking slave', $dp->as_string($dsn)); + + my $next_interval = $get_next_interval->(); + if ( time >= $next_interval ) { + do { $next_interval = $get_next_interval->() } + until $next_interval > time; + MKDEBUG && _d("Missed last interval; next interval:", + ts($next_interval)); + } + sleep $next_interval - time; + MKDEBUG && _d('Woke up at', ts(time)); + my ($delay, $hostname, $master_server_id) = $get_delay->($sth); - if ( $o->get('recurse') && $host ) { + if ( $o->get('recurse') ) { # Must print not only the delay, but the server's hostname if - # available. - printf("%-20s %5d\n", $host, $delay) unless $o->get('quiet'); + # available. Prefer the hostname from the DSN, then the hostname + # from @@hostname, then fall back to Socket or default File. + my $host = $dsn->{h} || $hostname || $dsn->{S} || $dsn->{F} || ''; + if ( $dsn->{P} && $dsn->{P} ne '3306' ) { + $host .= ":$dsn->{P}"; + } + printf($format_host, $host, $delay, $master_server_id); } else { # Just print the delay. - printf("%d\n", $delay) unless $o->get('quiet'); + printf($format_delay, $delay, $master_server_id); } } + return; } +# The interval iterator works by first returning the next whole second. +# So if the current time (since epoch) is 5.123, then the next whole second +# is 6.0, plus an optional skew. The next interval is 6.0 * the interval. +# If the interval is 0.5s, then the next interval is 6.5, plus an optional +# skew. Therefore, we always start on a whole second and return when the +# next interval is or should be. The caller can then sleep(time-next_interval) +# to wake up at that interval. If the caller misses the next interval, +# they just call the iterator until the next interval is later then the +# current time. +sub make_interval_iter { + my ( $interval, $skew ) = @_; + die "I need an interval argument" unless defined $interval; + my ($s) = gettimeofday(); + my $start_s = $s + 1; + my $i = 0; + my $get_next_interval = sub { + return $start_s + ($interval * $i++) + $skew; + }; + return $get_next_interval; +} + sub disconnect { my ( $dbh, $sth ) = @_; MKDEBUG && _d('Disconnecting'); @@ -2914,9 +3845,19 @@ use it to update a master or monitor a replica. If possible, MySQL connection options are read from your .my.cnf file. - mk-heartbeat -D test --update -h master-server - mk-heartbeat -D test --monitor -h slave-server - mk-heartbeat -D test --monitor -h slave-server --dbi-driver Pg +Start daemonized process to update test.heartbeat table on master: + + mk-heartbeat -D test --update -h master-server --daemonize + +Monitor replication lag on slave: + + mk-heartbeat -D test --monitor -h slave-server + + mk-heartbeat -D test --monitor -h slave-server --dbi-driver Pg + +Check slave lag once and exit (using optional DSN to specify slave host): + + mk-heartbeat -D test --check h=slave-server =head1 RISKS @@ -2945,18 +3886,29 @@ avoids reliance on the replication mechanism itself, which is unreliable. (For example, C on MySQL). -The first part is an instance of mk-heartbeat that connects to the master and -updates a timestamp ("heartbeat record") every second with L<"--update">. The -second part is another mk-heartbeat instance that connects to the slave, -examines the replicated heartbeat with L<"--monitor"> or L<"--check">, and -computes the difference from the current system time. If the slave's -replication is delayed or broken, the heartbeat will become stale. - -You must either manually create a heartbeat table on the master and insert one -row, or use L<"--create-table">. See L<"--create-table"> for the proper -heartbeat table structure. The C storage engine is suggested, but not +The first part is an L<"--update"> instance of mk-heartbeat that connects to +a master and updates a timestamp ("heartbeat record") every L<"--interval"> +seconds. Since the heartbeat table may contain records from multiple +masters (see L<"MULTI-SLAVE HIERARCHY">), the server's ID (@@server_id) is +used to identify records. + +The second part is a L<"--monitor"> or L<"--check"> instance of mk-heartbeat +that connects to a slave, examines the replicated heartbeat record from its +immediate master or the specified L<"--master-server-id">, and computes the +difference from the current system time. If replication between the slave and +the master is delayed or broken, the computed difference will be greater than +zero and potentially increase if L<"--monitor"> is specified. + +You must either manually create the heartbeat table on the master or use +L<"--create-table">. See L<"--create-table"> for the proper heartbeat +table structure. The C storage engine is suggested, but not required of course, for MySQL. +The heartbeat table must contain a heartbeat row. By default, a heartbeat +row is inserted if it doesn't exist. This feature can be disabled with the +L<"--[no]insert-heartbeat-row"> option in case the database user does not +have INSERT privileges. + mk-heartbeat depends only on the heartbeat record being replicated to the slave, so it works regardless of the replication mechanism (built-in replication, a system such as Continuent Tungsten, etc). It works at any depth in the @@ -2965,12 +3917,13 @@ to work and report (accurately!) that the slave is falling further and further behind the master. -mk-heartbeat has a one-second resolution. It depends on the clocks on the -master and slave servers being closely synchronized via NTP. L<"--update"> -checks happen on the edge of the second, and L<"--monitor"> checks happen -halfway between seconds. As long as the servers' clocks aren't skewed much -and the replication events are propagating in less than half a second, -mk-heartbeat will report zero seconds of delay. +mk-heartbeat has a maximum resolution of 0.01 second. The clocks on the +master and slave servers must be closely synchronized via NTP. By default, +L<"--update"> checks happen on the edge of the second (e.g. 00:01) and +L<"--monitor"> checks happen halfway between seconds (e.g. 00:01.5). +As long as the servers' clocks are closely synchronized and replication +events are propagating in less than half a second, mk-heartbeat will report +zero seconds of delay. mk-heartbeat will try to reconnect if the connection has an error, but will not retry if it can't get a connection when it first starts. @@ -2978,6 +3931,43 @@ The L<"--dbi-driver"> option lets you use mk-heartbeat to monitor PostgreSQL as well. It is reported to work well with Slony-1 replication. +=head1 MULTI-SLAVE HIERARCHY + +If the replication hierarchy has multiple slaves which are masters of +other slaves, like "master -> slave1 -> slave2", L<"--update"> instances +can be ran on the slaves as well as the master. The default heartbeat +table (see L<"--create-table">) is keyed on the C column, so +each server will update the row where C. + +For L<"--monitor"> and L<"--check">, if L<"--master-server-id"> is not +specified, the tool tries to discover and use the slave's immediate master. +If this fails, or if you want monitor lag from another master, then you can +specify the L<"--master-server-id"> to use. + +For example, if the replication hierarchy is "master -> slave1 -> slave2" +with corresponding server IDs 1, 2 and 3, you can: + + mk-heartbeat --daemonize -D test --update -h master + mk-heartbeat --daemonize -D test --update -h slave1 + +Then check (or monitor) the replication delay from master to slave2: + + mk-heartbeat -D test --master-server-id 1 --check slave2 + +Or check the replication delay from slave1 to slave2: + + mk-heartbeat -D test --master-server-id 2 --check slave2 + +Stopping the L<"--update"> instance one slave1 will not affect the instance +on master. + +=head1 MASTER AND SLAVE STATUS + +The default heartbeat table (see L<"--create-table">) has columns for saving +information from C and C. These +columns are optional. If any are present, their corresponding information +will be saved. + =head1 OPTIONS Specify at least one of L<"--stop">, L<"--update">, L<"--monitor">, or L<"--check">. @@ -3006,7 +3996,10 @@ =item --check -Check slave delay once and exit. +Check slave delay once and exit. If you also specify L<"--recurse">, the +tool will try to discover slave's of the given slave and check and print +their lag, too. The hostname or IP and port for each slave is printed +before its delay. L<"--recurse"> only works with MySQL. =item --config @@ -3022,17 +4015,41 @@ This option causes the table specified by L<"--database"> and L<"--table"> to be created with the following MAGIC_create_heartbeat table definition: - CREATE TABLE heartbeat ( - id int NOT NULL PRIMARY KEY, - ts datetime NOT NULL - ); + CREATE TABLE heartbeat ( + ts varchar(26) NOT NULL, + server_id int unsigned NOT NULL PRIMARY KEY, + file varchar(255) DEFAULT NULL, -- SHOW MASTER STATUS + position bigint unsigned DEFAULT NULL, -- SHOW MASTER STATUS + relay_master_log_file varchar(255) DEFAULT NULL, -- SHOW SLAVE STATUS + exec_master_log_pos bigint unsigned DEFAULT NULL -- SHOW SLAVE STATUS + ); The heartbeat table requires at least one row. If you manually create the heartbeat table, then you must insert a row by doing: - INSERT INTO heartbeat (id) VALUES (1); + INSERT INTO heartbeat (ts, server_id) VALUES (NOW(), N); + +where C is the server's ID; do not use @@server_id because it will replicate +and slaves will insert their own server ID instead of the master's server ID. + +This is done automatically by L<"--create-table">. + +A legacy version of the heartbeat table is still supported: + + CREATE TABLE heartbeat ( + id int NOT NULL PRIMARY KEY, + ts datetime NOT NULL + ); + +Legacy tables do not support L<"--update"> instances on each slave +of a multi-slave hierarchy like "master -> slave1 -> slave2". +To manually insert the one required row into a legacy table: + + INSERT INTO heartbeat (id, ts) VALUES (1, NOW()); + +The tool automatically detects if the heartbeat table is legacy. -This is done automatically by --create-table. +See also L<"MULTI-SLAVE HIERARCHY">. =item --daemonize @@ -3090,15 +4107,44 @@ Connect to host. +=item --[no]insert-heartbeat-row + +default: yes + +Insert a heartbeat row in the L<"--table"> if one doesn't exist. + +The heartbeat L<"--table"> requires a heartbeat row, else there's nothing +to L<"--update">, L<"--monitor">, or L<"--check">! By default, the tool will +insert a heartbeat row if one is not already present. You can disable this +feature by specifying C<--no-insert-heartbeat-row> in case the database user +does not have INSERT privileges. + =item --interval -type: time; default: 1s +type: float; default: 1.0 + +How often to update or check the heartbeat L<"--table">. Updates and checks +begin on the first whole second then repeat every L<"--interval"> seconds +for L<"--update"> and every L<"--interval"> plus L<"--skew"> seconds for +L<"--monitor">. + +For example, if at 00:00.4 an L<"--update"> instance is started at 0.5 second +intervals, the first update happens at 00:01.0, the next at 00:01.5, etc. +If at 00:10.7 a L<"--monitor"> instance is started at 0.05 second intervals +with the default 0.5 second L<"--skew">, then the first check happens at +00:11.5 (00:11.0 + 0.5) which will be L<"--skew"> seconds after the last update +which, because the instances are checking at synchronized intervals, happened +at 00:11.0. + +The tool waits for and begins on the first whole second just to make the +interval calculations simpler. Therefore, the tool could wait up to 1 second +before updating or checking. -Interval between updates and checks. +The minimum (fastest) interval is 0.01, and the maximum precision is two +decimal places, so 0.015 will be rounded to 0.02. -How often to check or update values. The updates and checks will happen when -the Unix time (seconds since epoch) is an even multiple of this value. The -suffix is similar to L<"--frames">. +If a legacy heartbeat table (see L<"--create-table">) is used, then the +maximum precision is 1s because the C column is type C. =item --log @@ -3106,6 +4152,14 @@ Print all output to this file when daemonized. +=item --master-server-id + +type: string + +Calculate delay from this master server ID for L<"--monitor"> or L<"--check">. +If not given, mk-heartbeat attempts to connect to the server's master and +determine its server id. + =item --monitor Monitor slave delay continuously. @@ -3138,11 +4192,11 @@ Port number to use for connection. -=item --quiet +=item --print-master-server-id -short form: -q - -Suppresses normal output. +Print the auto-detected or given L<"--master-server-id">. If L<"--check"> +or L<"--monitor"> is specified, specifying this option will print the +auto-detected or given L<"--master-server-id"> at the end of each line. =item --recurse @@ -3205,9 +4259,9 @@ =item --skew -type: int; default: 500000 +type: float; default: 0.5 -How long to delay checks, in milliseconds. +How long to delay checks. The default is to delay checks one half second. Since the update happens as soon as possible after the beginning of the second on the master, this allows @@ -3257,6 +4311,8 @@ Don't specify database.table; use L<"--database"> to specify the database. +See L<"--create-table">. + =item --update Update a master's heartbeat. @@ -3410,6 +4466,6 @@ =head1 VERSION -This manual page documents Ver 1.0.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.23 Distrib 7540 $Revision: 7537 $. =cut diff -Nru maatkit-7486/bin/mk-index-usage maatkit-7540/bin/mk-index-usage --- maatkit-7486/bin/mk-index-usage 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-index-usage 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.4'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -5928,6 +5928,6 @@ =head1 VERSION -This manual page documents Ver 0.9.4 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.4 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-kill maatkit-7540/bin/mk-kill --- maatkit-7486/bin/mk-kill 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-kill 2011-06-08 16:49:31.000000000 +0000 @@ -23,8 +23,8 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.10'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7482 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -2430,7 +2430,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -2704,35 +2704,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -2740,8 +2740,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -2783,7 +2783,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -2799,12 +2799,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -4841,6 +4841,6 @@ =head1 VERSION -This manual page documents Ver 0.9.10 Distrib 7486 $Revision: 7482 $. +This manual page documents Ver 0.9.10 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-loadavg maatkit-7540/bin/mk-loadavg --- maatkit-7486/bin/mk-loadavg 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-loadavg 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.7'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7460 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -4401,6 +4401,6 @@ =head1 VERSION -This manual page documents Ver 0.9.7 Distrib 7486 $Revision: 7460 $. +This manual page documents Ver 0.9.7 Distrib 7540 $Revision: 7460 $. =cut diff -Nru maatkit-7486/bin/mk-log-player maatkit-7540/bin/mk-log-player --- maatkit-7486/bin/mk-log-player 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-log-player 2011-06-08 16:49:31.000000000 +0000 @@ -21,8 +21,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.9'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -1046,7 +1046,7 @@ # ########################################################################### # ########################################################################### -# SlowLogParser package 7096 +# SlowLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SlowLogParser.pm @@ -1216,6 +1216,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # EVENT @@ -1239,7 +1243,7 @@ # ########################################################################### # ########################################################################### -# BinaryLogParser package 6785 +# BinaryLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/BinaryLogParser.pm @@ -1412,6 +1416,10 @@ if ( $found_arg ) { MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } else { @@ -1438,7 +1446,7 @@ # ########################################################################### # ########################################################################### -# GeneralLogParser package 7096 +# GeneralLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/GeneralLogParser.pm @@ -1572,6 +1580,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # LINE @@ -3620,6 +3632,6 @@ =head1 VERSION -This manual page documents Ver 1.0.9 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.9 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-merge-mqd-results maatkit-7540/bin/mk-merge-mqd-results --- maatkit-7486/bin/mk-merge-mqd-results 2011-05-05 19:59:29.000000000 +0000 +++ maatkit-7540/bin/mk-merge-mqd-results 2011-06-08 16:49:30.000000000 +0000 @@ -5639,6 +5639,6 @@ =head1 VERSION -This manual page documents Ver 0.9.28 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.29 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-parallel-dump maatkit-7540/bin/mk-parallel-dump --- maatkit-7486/bin/mk-parallel-dump 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-parallel-dump 2011-06-08 16:49:31.000000000 +0000 @@ -23,7 +23,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.28'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7460 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -5894,6 +5894,6 @@ =head1 VERSION -This manual page documents Ver 1.0.28 Distrib 7486 $Revision: 7460 $. +This manual page documents Ver 1.0.28 Distrib 7540 $Revision: 7460 $. =cut diff -Nru maatkit-7486/bin/mk-parallel-restore maatkit-7540/bin/mk-parallel-restore --- maatkit-7486/bin/mk-parallel-restore 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-parallel-restore 2011-06-08 16:49:31.000000000 +0000 @@ -23,7 +23,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.24'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -4018,6 +4018,6 @@ =head1 VERSION -This manual page documents Ver 1.0.24 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.24 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-profile-compact maatkit-7540/bin/mk-profile-compact --- maatkit-7486/bin/mk-profile-compact 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-profile-compact 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.1.22'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -1361,6 +1361,6 @@ =head1 VERSION -This manual page documents Ver 1.1.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.1.22 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-purge-logs maatkit-7540/bin/mk-purge-logs --- maatkit-7486/bin/mk-purge-logs 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-purge-logs 2011-06-08 16:49:31.000000000 +0000 @@ -21,8 +21,8 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.0'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -1403,7 +1403,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -1677,35 +1677,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -1713,8 +1713,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -1756,7 +1756,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -1772,12 +1772,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -2969,6 +2969,6 @@ =head1 VERSION -This manual page documents Ver 0.9.0 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.0 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-query-advisor maatkit-7540/bin/mk-query-advisor --- maatkit-7486/bin/mk-query-advisor 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-query-advisor 2011-06-08 16:49:31.000000000 +0000 @@ -21,8 +21,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.4'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # DSNParser package 7388 @@ -1481,7 +1481,7 @@ # ########################################################################### # ########################################################################### -# SlowLogParser package 7096 +# SlowLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SlowLogParser.pm @@ -1651,6 +1651,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # EVENT @@ -1674,7 +1678,7 @@ # ########################################################################### # ########################################################################### -# GeneralLogParser package 7096 +# GeneralLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/GeneralLogParser.pm @@ -1808,6 +1812,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # LINE @@ -4023,7 +4031,7 @@ # ########################################################################### # ########################################################################### -# SQLParser package 7433 +# SQLParser package 7497 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SQLParser.pm @@ -4070,6 +4078,7 @@ sub new { my ( $class, %args ) = @_; my $self = { + %args, }; return bless $self, $class; } @@ -4871,6 +4880,22 @@ else { die "Invalid number of parts in $type reference: $ident"; } + + if ( $self->{SchemaQualifier} ) { + if ( $type eq 'column' && !$ident_struct{tbl} ) { + my $qcol = $self->{SchemaQualifier}->qualify_column( + column => $ident_struct{col}, + ); + $ident_struct{db} = $qcol->{db} if $qcol->{db}; + $ident_struct{tbl} = $qcol->{tbl} if $qcol->{tbl}; + } + elsif ( $type eq 'table' && !$ident_struct{db} ) { + my $db = $self->{SchemaQualifier}->get_database_for_table( + table => $ident_struct{tbl}, + ); + $ident_struct{db} = $db if $db; + } + } MKDEBUG && _d($type, "identifier struct:", Dumper(\%ident_struct)); return \%ident_struct; @@ -4906,6 +4931,12 @@ return 0; } +sub set_SchemaQualifier { + my ( $self, $sq ) = @_; + $self->{SchemaQualifier} = $sq; + return; +} + sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } @@ -7244,6 +7275,6 @@ =head1 VERSION -This manual page documents Ver 1.0.4 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.4 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-query-digest maatkit-7540/bin/mk-query-digest --- maatkit-7486/bin/mk-query-digest 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-query-digest 2011-06-08 16:49:31.000000000 +0000 @@ -20,9 +20,9 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '0.9.28'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $VERSION = '0.9.29'; +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # DSNParser package 7388 @@ -2466,7 +2466,7 @@ # ########################################################################### # ########################################################################### -# TcpdumpParser package 6590 +# TcpdumpParser package 7505 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/TcpdumpParser.pm @@ -2515,6 +2515,8 @@ $packet->{pos_in_log} = $pos_in_log; $packet->{raw_packet} = $raw_packet; + $args{stats}->{events_read}++ if $args{stats}; + return $packet; } @@ -2597,7 +2599,7 @@ # ########################################################################### # ########################################################################### -# MySQLProtocolParser package 6909 +# MySQLProtocolParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MySQLProtocolParser.pm @@ -2973,6 +2975,7 @@ MKDEBUG && _d('Session deleted'); } + $args{stats}->{events_parsed}++ if $args{stats}; return $event; } @@ -4487,7 +4490,7 @@ # ########################################################################### # ########################################################################### -# SlowLogParser package 7096 +# SlowLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SlowLogParser.pm @@ -4657,6 +4660,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # EVENT @@ -8874,7 +8881,7 @@ # ########################################################################### # ########################################################################### -# MemcachedProtocolParser package 6590 +# MemcachedProtocolParser package 7521 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MemcachedProtocolParser.pm @@ -8914,6 +8921,12 @@ } my $packet = @args{@required_args}; + if ( $packet->{data_len} == 0 ) { + MKDEBUG && _d('No TCP data'); + $args{stats}->{no_tcp_data}++ if $args{stats}; + return; + } + my $src_host = "$packet->{src_host}:$packet->{src_port}"; my $dst_host = "$packet->{dst_host}:$packet->{dst_port}"; @@ -8921,6 +8934,7 @@ $server .= ":$self->{port}"; if ( $src_host ne $server && $dst_host ne $server ) { MKDEBUG && _d('Packet is not to or from', $server); + $args{stats}->{not_watched_server}++ if $args{stats}; return; } } @@ -8951,31 +8965,28 @@ }; my $session = $self->{sessions}->{$client}; - if ( $packet->{data_len} == 0 ) { - MKDEBUG && _d('No TCP data'); - return; - } - push @{$session->{raw_packets}}, $packet->{raw_packet}; $packet->{data} = pack('H*', $packet->{data}); my $event; if ( $packet_from eq 'server' ) { - $event = $self->_packet_from_server($packet, $session, $args{misc}); + $event = $self->_packet_from_server($packet, $session, %args); } elsif ( $packet_from eq 'client' ) { - $event = $self->_packet_from_client($packet, $session, $args{misc}); + $event = $self->_packet_from_client($packet, $session, %args); } else { + $args{stats}->{unknown_packet_origin}++ if $args{stats}; die 'Packet origin unknown'; } MKDEBUG && _d('Done with packet; event:', Dumper($event)); + $args{stats}->{events_parsed}++ if $args{stats}; return $event; } sub _packet_from_server { - my ( $self, $packet, $session, $misc ) = @_; + my ( $self, $packet, $session, %args ) = @_; die "I need a packet" unless $packet; die "I need a session" unless $session; @@ -8985,12 +8996,17 @@ if ( !$session->{state} ) { MKDEBUG && _d('Ignoring mid-stream server response'); + $args{stats}->{ignored_midstream_server_response}++ if $args{stats}; return; } if ( $session->{state} eq 'awaiting reply' ) { MKDEBUG && _d('State is awaiting reply'); my ($line1, $rest) = $packet->{data} =~ m/\A(.*?)\r\n(.*)?/s; + if ( !$line1 ) { + $args{stats}->{unknown_server_data}++ if $args{stats}; + die "Unknown memcached data from server"; + } my @vals = $line1 =~ m/(\S+)/g; $session->{res} = shift @vals; @@ -9009,11 +9025,12 @@ my ($key, $flags, $bytes) = @vals; defined $session->{flags} or $session->{flags} = $flags; defined $session->{bytes} or $session->{bytes} = $bytes; + if ( $rest && $bytes ) { MKDEBUG && _d('There is a value'); if ( length($rest) > $bytes ) { - MKDEBUG && _d('Looks like we got the whole response'); - $session->{val} = substr($rest, 0, $bytes); # Got the whole response. + MKDEBUG && _d('Got complete response'); + $session->{val} = substr($rest, 0, $bytes); } else { MKDEBUG && _d('Got partial response, saving for later'); @@ -9031,6 +9048,9 @@ elsif ( $session->{res} !~ m/STORED|DELETED|NOT_FOUND/ ) { MKDEBUG && _d('Unknown result'); } + else { + $args{stats}->{unknown_server_response}++ if $args{stats}; + } } else { # Should be 'partial recv' MKDEBUG && _d('Session state: ', $session->{state}); @@ -9060,7 +9080,7 @@ } sub _packet_from_client { - my ( $self, $packet, $session, $misc ) = @_; + my ( $self, $packet, $session, %args ) = @_; die "I need a packet" unless $packet; die "I need a session" unless $session; @@ -9082,6 +9102,12 @@ if ( !$session->{state} ) { MKDEBUG && _d('Session state: ', $session->{state}); ($line1, $val) = $packet->{data} =~ m/\A(.*?)\r\n(.+)?/s; + if ( !$line1 ) { + MKDEBUG && _d('Unknown memcached data from client, skipping packet'); + $args{stats}->{unknown_client_data}++ if $args{stats}; + return; + } + my @vals = $line1 =~ m/(\S+)/g; $cmd = lc shift @vals; MKDEBUG && _d('$cmd is a ', $cmd); @@ -9108,7 +9134,10 @@ } else { MKDEBUG && _d("Don't know how to handle", $cmd, "command"); + $args{stats}->{unknown_client_command}++ if $args{stats}; + return; } + @{$session}{qw(cmd key flags exptime)} = ($cmd, $key, $flags, $exptime); $session->{host} = $packet->{src_host}; @@ -9123,7 +9152,7 @@ $session->{state} = 'awaiting reply'; # Assume we got the whole packet if ( $val ) { if ( $session->{bytes} + 2 == length($val) ) { # +2 for the \r\n - MKDEBUG && _d('Got the whole thing'); + MKDEBUG && _d('Complete send'); $val =~ s/\r\n\Z//; # We got the whole thing. $session->{val} = $val; } @@ -9189,26 +9218,6 @@ return $errors_fh; } -sub fail_session { - my ( $self, $session, $reason ) = @_; - my $errors_fh = $self->_get_errors_fh(); - if ( $errors_fh ) { - $session->{reason_for_failure} = $reason; - my $session_dump = '# ' . Dumper($session); - chomp $session_dump; - $session_dump =~ s/\n/\n# /g; - print $errors_fh "$session_dump\n"; - { - local $LIST_SEPARATOR = "\n"; - print $errors_fh "@{$session->{raw_packets}}"; - print $errors_fh "\n"; - } - } - MKDEBUG && _d('Failed session', $session->{client}, 'because', $reason); - delete $self->{sessions}->{$session->{client}}; - return; -} - sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } @@ -9388,7 +9397,7 @@ # ########################################################################### # ########################################################################### -# BinaryLogParser package 6785 +# BinaryLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/BinaryLogParser.pm @@ -9561,6 +9570,10 @@ if ( $found_arg ) { MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } else { @@ -9587,7 +9600,7 @@ # ########################################################################### # ########################################################################### -# GeneralLogParser package 7096 +# GeneralLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/GeneralLogParser.pm @@ -9721,6 +9734,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # LINE @@ -9744,7 +9761,7 @@ # ########################################################################### # ########################################################################### -# ProtocolParser package 7096 +# ProtocolParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/ProtocolParser.pm @@ -9809,11 +9826,13 @@ my $event; map { $event = $self->_parse_packet($_, $args{misc}); + $args{stats}->{events_parsed}++ if $args{stats}; } sort { $a->{seq} <=> $b->{seq} } @{$session->{client_packets}}; map { $event = $self->_parse_packet($_, $args{misc}); + $args{stats}->{events_parsed}++ if $args{stats}; } sort { $a->{seq} <=> $b->{seq} } @{$session->{server_packets}}; @@ -9825,7 +9844,9 @@ return; } - return $self->_parse_packet($packet, $args{misc}); + my $event = $self->_parse_packet($packet, $args{misc}); + $args{stats}->{events_parsed}++ if $args{stats}; + return $event; } sub _parse_packet { @@ -10390,7 +10411,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -10664,35 +10685,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); + + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -10700,8 +10721,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -10743,7 +10764,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -10759,12 +10780,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -11747,7 +11768,7 @@ # ########################################################################### # ########################################################################### -# Pipeline package 7268 +# Pipeline package 7509 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/Pipeline.pm @@ -11829,6 +11850,8 @@ my $pipeline_data = $args{pipeline_data} || {}; $pipeline_data->{oktorun} = $oktorun; + my $stats = $args{stats}; # optional + MKDEBUG && _d("Pipeline starting at", time); my $instrument = $self->{instrument}; my $processes = $self->{procs}; @@ -11855,6 +11878,10 @@ if ( !$output ) { MKDEBUG && _d("Pipeline restarting early after", $self->{names}->[$procno]); + if ( $stats ) { + $stats->{"pipeline_restarted_after_" + .$self->{names}->[$procno]}++; + } last PIPELINE_PROCESS; } $procno++; @@ -12217,7 +12244,10 @@ # pass data through the pipeline. The most importat data is the event. # Other data includes in the next_event callback, time and iters left, # etc. This hashref is accessed inside a proc via the $args arg. - my $pipeline_data = { iter => 1 }; + my $pipeline_data = { + iter => 1, + stats => \%stats, + }; # Enable timings to instrument code for either of these two opts. # Else, don't instrument to avoid cost of measurement. @@ -12436,6 +12466,7 @@ tell => $args->{tell}, misc => $args->{misc}, oktorun => sub { $args->{more_events} = $_[0]; }, + stats => $args->{stats}, ); if ( $event ) { $args->{event} = $event; @@ -12669,14 +12700,18 @@ if ( $report ) { MKDEBUG && _d("Iteration", $args->{iter}, "stopped at",ts(time)); - if ( $ea[0]->events_processed() ) { + + # Get this before calling print_reports() because that sub + # resets each ea and we may need this later for stats. + my $n_events_aggregated = $ea[0]->events_processed(); + + if ( $n_events_aggregated ) { print_reports( eas => \@ea, tls => \@tl, groupby => \@groupby, orderby => \@orderby, files => \@read_files, - stats => \%stats, Pipeline => $pipeline, QueryReview => $qv, ExplainAnalyzer => $exa, @@ -12687,6 +12722,54 @@ print "\n# No events processed.\n"; } + if ( $o->get('statistics') ) { + if ( keys %stats ) { + my $report = new ReportFormatter( + line_width => 74, + ); + $report->set_columns( + { name => 'Statistic', }, + { name => 'Count', right_justify => 1 }, + { name => '%/Events', right_justify => 1 }, + ); + + # Have to add this one manually because currently + # EventAggregator::aggregate() doesn't know about stats. + # It's the same thing as events_processed() though. + $stats{events_aggregated} = $n_events_aggregated; + + # Save value else events_read will be reset during the + # foreach loop below and mess up percentage_of(). + my $n_events_read = $stats{events_read} || 0; + + my %stats_sort_order = ( + events_read => 1, + events_parsed => 2, + events_aggregated => 3, + ); + my @stats = sort { + QueryReportFormatter::pref_sort( + $a, $stats_sort_order{$a}, + $b, $stats_sort_order{$b}) + } keys %stats; + foreach my $stat ( @stats ) { + $report->add_line( + $stat, + $stats{$stat} || 0, + percentage_of( + $stats{$stat} || 0, + $n_events_read, + p => 2), + ); + $stats{$stat} = 0; # Reset for next iteration. + } + print "\n" . $report->get_report(); + } + else { + print "\n# No statistics values.\n"; + } + } + # Decrement iters_left after finishing an iter because in the # default case, 1 iter, if we decr when the iter starts, then # terminator will think there's no iters left before the one @@ -13187,6 +13270,7 @@ $pipeline->execute( oktorun => \$oktorun, pipeline_data => $pipeline_data, + stats => \%stats, ); }; if ( $EVAL_ERROR ) { @@ -13322,7 +13406,7 @@ ); } - $eas->[$i]->reset_aggregated_data(); # reset for next iteration + $eas->[$i]->reset_aggregated_data(); # Reset for next iteration. # Print header report only once. So remove it from the # list of reports after the first groupby's reports. @@ -13356,28 +13440,6 @@ print "\n" . $report->get_report(); } - if ( $o->get('statistics') ) { - if ( keys %$stats ) { - my $report = new ReportFormatter( - line_width => 74, - ); - $report->set_columns( - { name => 'Statistic', }, - { name => 'Value', right_justify => 1 }, - ); - foreach my $stat ( sort keys %$stats ) { - $report->add_line($stat, $stats->{$stat} || 0); - - # Reset stats for next iteration. - $stats->{$stat} = 0; - } - print "\n" . $report->get_report(); - } - else { - print "\n# No statistics values.\n"; - } - } - return; } @@ -15392,7 +15454,36 @@ =item --statistics -Print statistics. +Print statistics about internal counters. This option is mostly for +development and debugging. The statistics report is printed for each +iteration after all other reports, even if no events are processed or +C<--no-report> is specified. The statistics report looks like: + + # No events processed. + + # Statistic Count %/Events + # ================================================ ====== ======== + # events_read 142030 100.00 + # events_parsed 50430 35.51 + # events_aggregated 0 0.00 + # ignored_midstream_server_response 18111 12.75 + # no_tcp_data 91600 64.49 + # pipeline_restarted_after_MemcachedProtocolParser 142030 100.00 + # pipeline_restarted_after_TcpdumpParser 1 0.00 + # unknown_client_command 1 0.00 + # unknown_client_data 32318 22.75 + +The first column is the internal counter name; the second column is counter's +count; and the third column is the count as a percentage of C. + +In this case, it shows why no events were processed/aggregated: 100% of events +were rejected by the C. Of those, 35.51% were data +packets, but of these 12.75% of ignored mid-stream server response, one was +an unknown client command, and 22.75% were unknown client data. The other +64.49% were TCP control packets (probably most ACKs). + +Since mk-query-digest is complex, you will probably need someone familiar +with its code to decipher the statistics report. =item --table-access @@ -15808,6 +15899,6 @@ =head1 VERSION -This manual page documents Ver 0.9.28 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.29 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-query-profiler maatkit-7540/bin/mk-query-profiler --- maatkit-7486/bin/mk-query-profiler 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-query-profiler 2011-06-08 16:49:31.000000000 +0000 @@ -23,7 +23,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.1.22'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -2855,6 +2855,6 @@ =head1 VERSION -This manual page documents Ver 1.1.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.1.22 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-show-grants maatkit-7540/bin/mk-show-grants --- maatkit-7486/bin/mk-show-grants 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-show-grants 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ # Place, Suite 330, Boston, MA 02111-1307 USA. our $VERSION = '1.0.23'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); use strict; @@ -2229,6 +2229,6 @@ =head1 VERSION -This manual page documents Ver 1.0.23 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.23 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-slave-delay maatkit-7540/bin/mk-slave-delay --- maatkit-7486/bin/mk-slave-delay 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-slave-delay 2011-06-08 16:49:31.000000000 +0000 @@ -23,7 +23,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.23'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -2705,6 +2705,6 @@ =head1 VERSION -This manual page documents Ver 1.0.23 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.23 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-slave-find maatkit-7540/bin/mk-slave-find --- maatkit-7486/bin/mk-slave-find 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-slave-find 2011-06-08 16:49:31.000000000 +0000 @@ -24,8 +24,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.16'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7480 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -1406,7 +1406,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -1680,35 +1680,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -1716,8 +1716,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -1759,7 +1759,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -1775,12 +1775,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -3353,6 +3353,6 @@ =head1 VERSION -This manual page documents Ver 1.0.16 Distrib 7486 $Revision: 7480 $. +This manual page documents Ver 1.0.16 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-slave-move maatkit-7540/bin/mk-slave-move --- maatkit-7486/bin/mk-slave-move 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-slave-move 2011-06-08 16:49:31.000000000 +0000 @@ -24,8 +24,8 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.12'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -1406,7 +1406,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -1680,35 +1680,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -1716,8 +1716,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -1759,7 +1759,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -1775,12 +1775,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -2937,6 +2937,6 @@ =head1 VERSION -This manual page documents Ver 0.9.12 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.12 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-slave-prefetch maatkit-7540/bin/mk-slave-prefetch --- maatkit-7486/bin/mk-slave-prefetch 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-slave-prefetch 2011-06-08 16:49:31.000000000 +0000 @@ -23,8 +23,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.21'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); use English qw(-no_match_vars); $OUTPUT_AUTOFLUSH = 1; @@ -1492,7 +1492,7 @@ # ########################################################################### # ########################################################################### -# BinaryLogParser package 6785 +# BinaryLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/BinaryLogParser.pm @@ -1665,6 +1665,10 @@ if ( $found_arg ) { MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } else { @@ -5422,6 +5426,6 @@ =head1 VERSION -This manual page documents Ver 1.0.21 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.21 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-slave-restart maatkit-7540/bin/mk-slave-restart --- maatkit-7486/bin/mk-slave-restart 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-slave-restart 2011-06-08 16:49:31.000000000 +0000 @@ -24,8 +24,8 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.22'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # Quoter package 6850 @@ -1568,7 +1568,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -1842,35 +1842,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } + + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -1878,8 +1878,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -1921,7 +1921,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -1937,12 +1937,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -3532,6 +3532,6 @@ =head1 VERSION -This manual page documents Ver 1.0.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.22 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-table-checksum maatkit-7540/bin/mk-table-checksum --- maatkit-7486/bin/mk-table-checksum 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-table-checksum 2011-06-08 16:49:31.000000000 +0000 @@ -22,9 +22,9 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '1.2.22'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7480 $ =~ m/(\d+)/g, 0)); +our $VERSION = '1.2.23'; +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7527 $ =~ m/(\d+)/g, 0)); # ########################################################################### # TableParser package 7156 @@ -3583,7 +3583,7 @@ # ########################################################################### # ########################################################################### -# MasterSlave package 6935 +# MasterSlave package 7525 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/MasterSlave.pm @@ -3857,35 +3857,35 @@ sub get_master_status { my ( $self, $dbh ) = @_; - if ( !$self->{not_a_master}->{$dbh} ) { - my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} - ||= $dbh->prepare('SHOW MASTER STATUS'); - MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); - $sth->execute(); - my ($ms) = @{$sth->fetchall_arrayref({})}; - if ( $ms && %$ms ) { - $ms = { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys - if ( $ms->{file} && $ms->{position} ) { - return $ms; - } - } + if ( $self->{not_a_master}->{$dbh} ) { + MKDEBUG && _d('Server on dbh', $dbh, 'is not a master'); + return; + } - MKDEBUG && _d('This server returns nothing for SHOW MASTER STATUS'); + my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} + ||= $dbh->prepare('SHOW MASTER STATUS'); + MKDEBUG && _d($dbh, 'SHOW MASTER STATUS'); + $sth->execute(); + my ($ms) = @{$sth->fetchall_arrayref({})}; + MKDEBUG && _d(Dumper($ms)); + + if ( !$ms || scalar keys %$ms < 2 ) { + MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); $self->{not_a_master}->{$dbh}++; } + + return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys } sub wait_for_master { my ( $self, %args ) = @_; - my @required_args = qw(master_dbh slave_dbh); + my @required_args = qw(master_status slave_dbh); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($master_dbh, $slave_dbh) = @args{@required_args}; + my ($master_status, $slave_dbh) = @args{@required_args}; my $timeout = $args{timeout} || 60; - my $master_status = $args{master_status} - || $self->get_master_status($master_dbh); my $result; my $waited; @@ -3893,8 +3893,8 @@ my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " . "$master_status->{position}, $timeout)"; MKDEBUG && _d($slave_dbh, $sql); - my $start = time; - ($result) = $slave_dbh->selectrow_array($sql); + my $start = time; + ($result) = $slave_dbh->selectrow_array($sql); $waited = time - $start; @@ -3936,7 +3936,7 @@ } sub catchup_to_master { - my ( $self, $slave, $master, $time ) = @_; + my ( $self, $slave, $master, $timeout ) = @_; $self->stop_slave($master); $self->stop_slave($slave); my $slave_status = $self->get_slave_status($slave); @@ -3952,12 +3952,12 @@ $self->start_slave($slave, $master_pos); $result = $self->wait_for_master( - master_dbh => $master, + master_status => $master_status, slave_dbh => $slave, - timeout => $time, + timeout => $timeout, master_status => $master_status ); - if ( !defined $result ) { + if ( !defined $result->{result} ) { $slave_status = $self->get_slave_status($slave); if ( !$self->slave_is_running($slave_status) ) { MKDEBUG && _d('Master position:', @@ -4549,7 +4549,7 @@ # ########################################################################### # ########################################################################### -# SchemaIterator package 7141 +# SchemaIterator package 7512 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SchemaIterator.pm @@ -4558,287 +4558,354 @@ # ########################################################################### package SchemaIterator; +{ # package scope use strict; use warnings FATAL => 'all'; - use English qw(-no_match_vars); +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + use Data::Dumper; $Data::Dumper::Indent = 1; $Data::Dumper::Sortkeys = 1; $Data::Dumper::Quotekeys = 0; -use constant MKDEBUG => $ENV{MKDEBUG} || 0; +my $open_comment = qr{/\*!\d{5} }; +my $tbl_name = qr{ + CREATE\s+ + (?:TEMPORARY\s+)? + TABLE\s+ + (?:IF NOT EXISTS\s+)? + ([^\(]+) +}x; + sub new { my ( $class, %args ) = @_; - foreach my $arg ( qw(Quoter) ) { + my @required_args = qw(OptionParser Quoter); + foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } + + my ($file_itr, $dbh) = @args{qw(file_itr dbh)}; + die "I need either a dbh or file_itr argument" + if (!$dbh && !$file_itr) || ($dbh && $file_itr); + my $self = { %args, - filter => undef, - dbs => [], + filters => _make_filters(%args), }; + return bless $self, $class; } -sub make_filter { - my ( $self, $o ) = @_; - my @lines = ( - 'sub {', - ' my ( $dbh, $db, $tbl ) = @_;', - ' my $engine = undef;', - ); +sub _make_filters { + my ( %args ) = @_; + my @required_args = qw(OptionParser Quoter); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($o, $q) = @args{@required_args}; + my %filters; - my @permit_dbs = _make_filter('unless', '$db', $o->get('databases')) - if $o->has('databases'); - my @reject_dbs = _make_filter('if', '$db', $o->get('ignore-databases')) - if $o->has('ignore-databases'); - my @dbs_regex; - if ( $o->has('databases-regex') && (my $p = $o->get('databases-regex')) ) { - push @dbs_regex, " return 0 unless \$db && (\$db =~ m/$p/o);"; - } - my @reject_dbs_regex; - if ( $o->has('ignore-databases-regex') - && (my $p = $o->get('ignore-databases-regex')) ) { - push @reject_dbs_regex, " return 0 if \$db && (\$db =~ m/$p/o);"; - } - if ( @permit_dbs || @reject_dbs || @dbs_regex || @reject_dbs_regex ) { - push @lines, - ' if ( $db ) {', - (@permit_dbs ? @permit_dbs : ()), - (@reject_dbs ? @reject_dbs : ()), - (@dbs_regex ? @dbs_regex : ()), - (@reject_dbs_regex ? @reject_dbs_regex : ()), - ' }'; - } - - if ( $o->has('tables') || $o->has('ignore-tables') - || $o->has('ignore-tables-regex') ) { - - my $have_qtbl = 0; - my $have_only_qtbls = 0; - my %qtbls; - - my @permit_tbls; - my @permit_qtbls; - my %permit_qtbls; - if ( $o->get('tables') ) { - my %tbls; - map { - if ( $_ =~ m/\./ ) { - $permit_qtbls{$_} = 1; + + my @simple_filters = qw( + databases tables engines + ignore-databases ignore-tables ignore-engines); + FILTER: + foreach my $filter ( @simple_filters ) { + if ( $o->has($filter) ) { + my $objs = $o->get($filter); + next FILTER unless $objs && scalar keys %$objs; + my $is_table = $filter =~ m/table/ ? 1 : 0; + foreach my $obj ( keys %$objs ) { + die "Undefined value for --$filter" unless $obj; + $obj = lc $obj; + if ( $is_table ) { + my ($db, $tbl) = $q->split_unquote($obj); + $db ||= '*'; + MKDEBUG && _d('Filter', $filter, 'value:', $db, $tbl); + $filters{$filter}->{$tbl} = $db; } - else { - $tbls{$_} = 1; + else { # database + MKDEBUG && _d('Filter', $filter, 'value:', $obj); + $filters{$filter}->{$obj} = 1; } - } keys %{ $o->get('tables') }; - @permit_tbls = _make_filter('unless', '$tbl', \%tbls); - @permit_qtbls = _make_filter('unless', '$qtbl', \%permit_qtbls); + } + } + } + + my @regex_filters = qw( + databases-regex tables-regex + ignore-databases-regex ignore-tables-regex); + REGEX_FILTER: + foreach my $filter ( @regex_filters ) { + if ( $o->has($filter) ) { + my $pat = $o->get($filter); + next REGEX_FILTER unless $pat; + $filters{$filter} = qr/$pat/; + MKDEBUG && _d('Filter', $filter, 'value:', $filters{$filter}); + } + } + + MKDEBUG && _d('Schema object filters:', Dumper(\%filters)); + return \%filters; +} + +sub next_schema_object { + my ( $self ) = @_; + + my %schema_object; + if ( $self->{file_itr} ) { + %schema_object = $self->_iterate_files(); + } + else { # dbh + %schema_object = $self->_iterate_dbh(); + } + + MKDEBUG && _d('Next schema object:', Dumper(\%schema_object)); + return %schema_object; +} + +sub _iterate_files { + my ( $self ) = @_; - if ( @permit_qtbls ) { - push @lines, - ' my $qtbl = ($db ? "$db." : "") . ($tbl ? $tbl : "");'; - $have_qtbl = 1; + if ( !$self->{fh} ) { + my ($fh, $file) = $self->{file_itr}->(); + if ( !$fh ) { + MKDEBUG && _d('No more files to iterate'); + return; + } + $self->{fh} = $fh; + $self->{file} = $file; + } + my $fh = $self->{fh}; + MKDEBUG && _d('Getting next schema object from', $self->{file}); + + local $INPUT_RECORD_SEPARATOR = ''; + CHUNK: + while (defined(my $chunk = <$fh>)) { + if ($chunk =~ m/Database: (\S+)/) { + my $db = $1; # XXX + $db =~ s/^`//; # strip leading ` + $db =~ s/`$//; # and trailing ` + if ( $self->database_is_allowed($db) ) { + $self->{db} = $db; } } + elsif ($self->{db} && $chunk =~ m/CREATE TABLE/) { + if ($chunk =~ m/DROP VIEW IF EXISTS/) { + MKDEBUG && _d('Table is a VIEW, skipping'); + next CHUNK; + } - my @reject_tbls; - my @reject_qtbls; - my %reject_qtbls; - if ( $o->get('ignore-tables') ) { - my %tbls; - map { - if ( $_ =~ m/\./ ) { - $reject_qtbls{$_} = 1; - } - else { - $tbls{$_} = 1; + my ($tbl) = $chunk =~ m/$tbl_name/; + $tbl =~ s/^\s*`//; + $tbl =~ s/`\s*$//; + if ( $self->table_is_allowed($self->{db}, $tbl) ) { + my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms; + if ( !$ddl ) { + warn "Failed to parse CREATE TABLE from\n" . $chunk; + next CHUNK; } - } keys %{ $o->get('ignore-tables') }; - @reject_tbls= _make_filter('if', '$tbl', \%tbls); - @reject_qtbls = _make_filter('if', '$qtbl', \%reject_qtbls); + $ddl =~ s/ \*\/;\Z/;/; # remove end of version comment + + my ($engine) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; - if ( @reject_qtbls && !$have_qtbl ) { - push @lines, - ' my $qtbl = ($db ? "$db." : "") . ($tbl ? $tbl : "");'; + if ( !$engine || $self->engine_is_allowed($engine) ) { + return ( + db => $self->{db}, + tbl => $tbl, + ddl => $ddl, + ); + } } } + } # CHUNK - if ( keys %permit_qtbls && !@permit_dbs ) { - my $dbs = {}; - map { - my ($db, undef) = split(/\./, $_); - $dbs->{$db} = 1; - } keys %permit_qtbls; - MKDEBUG && _d('Adding restriction "--databases', - (join(',', keys %$dbs) . '"')); - if ( keys %$dbs ) { - $o->set('databases', $dbs); - return $self->make_filter($o); - } - } - - my @tbls_regex; - if ( $o->has('tables-regex') && (my $p = $o->get('tables-regex')) ) { - push @tbls_regex, " return 0 unless \$tbl && (\$tbl =~ m/$p/o);"; - } - my @reject_tbls_regex; - if ( $o->has('ignore-tables-regex') - && (my $p = $o->get('ignore-tables-regex')) ) { - push @reject_tbls_regex, - " return 0 if \$tbl && (\$tbl =~ m/$p/o);"; - } - - my @get_eng; - my @permit_engs; - my @reject_engs; - if ( ($o->has('engines') && $o->get('engines')) - || ($o->has('ignore-engines') && $o->get('ignore-engines')) ) { - push @get_eng, - ' my $sql = "SHOW TABLE STATUS "', - ' . ($db ? "FROM `$db`" : "")', - ' . " LIKE \'$tbl\'";', - ' MKDEBUG && _d($sql);', - ' eval {', - ' $engine = $dbh->selectrow_hashref($sql)->{engine};', - ' };', - ' MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);', - ' MKDEBUG && _d($tbl, "uses engine", $engine);', - ' $engine = lc $engine if $engine;', - @permit_engs - = _make_filter('unless', '$engine', $o->get('engines'), 1); - @reject_engs - = _make_filter('if', '$engine', $o->get('ignore-engines'), 1) - } - - if ( @permit_tbls || @permit_qtbls || @reject_tbls || @tbls_regex - || @reject_tbls_regex || @permit_engs || @reject_engs ) { - push @lines, - ' if ( $tbl ) {', - (@permit_tbls ? @permit_tbls : ()), - (@reject_tbls ? @reject_tbls : ()), - (@tbls_regex ? @tbls_regex : ()), - (@reject_tbls_regex ? @reject_tbls_regex : ()), - (@permit_qtbls ? @permit_qtbls : ()), - (@reject_qtbls ? @reject_qtbls : ()), - (@get_eng ? @get_eng : ()), - (@permit_engs ? @permit_engs : ()), - (@reject_engs ? @reject_engs : ()), - ' }'; - } - } - - push @lines, - ' MKDEBUG && _d(\'Passes filters:\', $db, $tbl, $engine, $dbh);', - ' return 1;', '}'; - - my $code = join("\n", @lines); - MKDEBUG && _d('filter sub:', $code); - my $filter_sub= eval $code - or die "Error compiling subroutine code:\n$code\n$EVAL_ERROR"; - - return $filter_sub; -} - -sub set_filter { - my ( $self, $filter_sub ) = @_; - $self->{filter} = $filter_sub; - MKDEBUG && _d('Set filter sub'); - return; + MKDEBUG && _d('No more schema objects in', $self->{file}); + close $self->{fh}; + $self->{fh} = undef; + + return $self->_iterate_files(); } -sub get_db_itr { - my ( $self, %args ) = @_; - my @required_args = qw(dbh); - foreach my $arg ( @required_args ) { - die "I need a $arg argument" unless $args{$arg}; - } - my ($dbh) = @args{@required_args}; +sub _iterate_dbh { + my ( $self ) = @_; + my $q = $self->{Quoter}; + my $dbh = $self->{dbh}; + MKDEBUG && _d('Getting next schema object from dbh', $dbh); - my $filter = $self->{filter}; - my @dbs; - eval { + if ( !defined $self->{dbs} ) { my $sql = 'SHOW DATABASES'; MKDEBUG && _d($sql); - @dbs = grep { - my $ok = $filter ? $filter->($dbh, $_, undef) : 1; - $ok = 0 if $_ =~ m/information_schema|performance_schema|lost\+found/; - $ok; - } @{ $dbh->selectcol_arrayref($sql) }; + my @dbs = grep { $self->database_is_allowed($_) } + @{$dbh->selectcol_arrayref($sql)}; MKDEBUG && _d('Found', scalar @dbs, 'databases'); - }; + $self->{dbs} = \@dbs; + } - MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); - my $iterator = sub { - return shift @dbs; - }; + if ( !$self->{db} ) { + $self->{db} = shift @{$self->{dbs}}; + MKDEBUG && _d('Next database:', $self->{db}); + return unless $self->{db}; + } - if (wantarray) { - return ($iterator, scalar @dbs); + if ( !defined $self->{tbls} ) { + my $sql = 'SHOW /*!50002 FULL*/ TABLES FROM ' . $q->quote($self->{db}); + MKDEBUG && _d($sql); + my @tbls = map { + $_->[0]; # (tbl, type) + } + grep { + my ($tbl, $type) = @$_; + $self->table_is_allowed($self->{db}, $tbl) + && (!$type || ($type ne 'VIEW')); + } + @{$dbh->selectall_arrayref($sql)}; + MKDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db}); + $self->{tbls} = \@tbls; } - else { - return $iterator; + + while ( my $tbl = shift @{$self->{tbls}} ) { + my $engine; + if ( $self->{filters}->{'engines'} + || $self->{filters}->{'ignore-engines'} ) { + my $sql = "SHOW TABLE STATUS FROM " . $q->quote($self->{db}) + . " LIKE \'$tbl\'"; + MKDEBUG && _d($sql); + $engine = $dbh->selectrow_hashref($sql)->{engine}; + MKDEBUG && _d($tbl, 'uses', $engine, 'engine'); + } + + + if ( !$engine || $self->engine_is_allowed($engine) ) { + my $ddl; + if ( my $du = $self->{MySQLDump} ) { + $ddl = $du->get_create_table($dbh, $q, $self->{db}, $tbl)->[1]; + } + + return ( + db => $self->{db}, + tbl => $tbl, + ddl => $ddl, + ); + } } + + MKDEBUG && _d('No more tables in database', $self->{db}); + $self->{db} = undef; + $self->{tbls} = undef; + + return $self->_iterate_dbh(); } -sub get_tbl_itr { - my ( $self, %args ) = @_; - my @required_args = qw(dbh db); - foreach my $arg ( @required_args ) { - die "I need a $arg argument" unless $args{$arg}; +sub database_is_allowed { + my ( $self, $db ) = @_; + die "I need a db argument" unless $db; + + $db = lc $db; + + my $filter = $self->{filters}; + + if ( $db =~ m/information_schema|performance_schema|lost\+found/ ) { + MKDEBUG && _d('Database', $db, 'is a system database, ignoring'); + return 0; } - my ($dbh, $db, $views) = @args{@required_args, 'views'}; - my $filter = $self->{filter}; - my @tbls; - if ( $db ) { - eval { - my $sql = 'SHOW /*!50002 FULL*/ TABLES FROM ' - . $self->{Quoter}->quote($db); - MKDEBUG && _d($sql); - @tbls = map { - $_->[0] - } - grep { - my ($tbl, $type) = @$_; - my $ok = $filter ? $filter->($dbh, $db, $tbl) : 1; - if ( !$views ) { - $ok = 0 if ($type || '') eq 'VIEW'; - } - $ok; - } - @{ $dbh->selectall_arrayref($sql) }; - MKDEBUG && _d('Found', scalar @tbls, 'tables in', $db); - }; - MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); + if ( $self->{filters}->{'ignore-databases'}->{$db} ) { + MKDEBUG && _d('Database', $db, 'is in --ignore-databases list'); + return 0; } - else { - MKDEBUG && _d('No db given so no tables'); + + if ( $filter->{'ignore-databases-regex'} + && $db =~ $filter->{'ignore-databases-regex'} ) { + MKDEBUG && _d('Database', $db, 'matches --ignore-databases-regex'); + return 0; } - my $iterator = sub { - return shift @tbls; - }; + if ( $filter->{'databases'} + && !$filter->{'databases'}->{$db} ) { + MKDEBUG && _d('Database', $db, 'is not in --databases list, ignoring'); + return 0; + } - if ( wantarray ) { - return ($iterator, scalar @tbls); + if ( $filter->{'databases-regex'} + && $db !~ $filter->{'databases-regex'} ) { + MKDEBUG && _d('Database', $db, 'does not match --databases-regex, ignoring'); + return 0; } - else { - return $iterator; + + return 1; +} + +sub table_is_allowed { + my ( $self, $db, $tbl ) = @_; + die "I need a db argument" unless $db; + die "I need a tbl argument" unless $tbl; + + $db = lc $db; + $tbl = lc $tbl; + + my $filter = $self->{filters}; + + if ( $filter->{'ignore-tables'}->{$tbl} + && ($filter->{'ignore-tables'}->{$tbl} eq '*' + || $filter->{'ignore-tables'}->{$tbl} eq $db) ) { + MKDEBUG && _d('Table', $tbl, 'is in --ignore-tables list'); + return 0; + } + + if ( $filter->{'ignore-tables-regex'} + && $tbl =~ $filter->{'ignore-tables-regex'} ) { + MKDEBUG && _d('Table', $tbl, 'matches --ignore-tables-regex'); + return 0; + } + + if ( $filter->{'tables'} + && !$filter->{'tables'}->{$tbl} ) { + MKDEBUG && _d('Table', $tbl, 'is not in --tables list, ignoring'); + return 0; + } + + if ( $filter->{'tables-regex'} + && $tbl !~ $filter->{'tables-regex'} ) { + MKDEBUG && _d('Table', $tbl, 'does not match --tables-regex, ignoring'); + return 0; + } + + if ( $filter->{'tables'} + && $filter->{'tables'}->{$tbl} + && $filter->{'tables'}->{$tbl} ne '*' + && $filter->{'tables'}->{$tbl} ne $db ) { + MKDEBUG && _d('Table', $tbl, 'is only allowed in database', + $filter->{'tables'}->{$tbl}); + return 0; } + + return 1; } -sub _make_filter { - my ( $cond, $var_name, $objs, $lc ) = @_; - my @lines; - if ( scalar keys %$objs ) { - my $test = join(' || ', - map { "$var_name eq '" . ($lc ? lc $_ : $_) ."'" } keys %$objs); - push @lines, " return 0 $cond $var_name && ($test);", +sub engine_is_allowed { + my ( $self, $engine ) = @_; + die "I need an engine argument" unless $engine; + + $engine = lc $engine; + + my $filter = $self->{filters}; + + if ( $filter->{'ignore-engines'}->{$engine} ) { + MKDEBUG && _d('Engine', $engine, 'is in --ignore-databases list'); + return 0; } - return @lines; + + if ( $filter->{'engines'} + && !$filter->{'engines'}->{$engine} ) { + MKDEBUG && _d('Engine', $engine, 'is not in --engines list, ignoring'); + return 0; + } + + return 1; } sub _d { @@ -4849,6 +4916,7 @@ print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } +} # package scope 1; # ########################################################################### @@ -5235,6 +5303,13 @@ . 'Use --resume-replicate instead.'); } + if ( my $throttle_method = $o->get('throttle-method') ) { + $throttle_method = lc $throttle_method; + if ( !grep { $throttle_method eq $_ } qw(none slavelag) ) { + $o->save_error("Invalid --throttle-method: $throttle_method"); + } + } + if ( $o->get('check-slave-lag') && $o->get('throttle-method') eq 'none') { # User specified --check-slave-lag DSN and --throttle-method none. # They probably meant just --check-slave-lag DSN. @@ -5325,7 +5400,7 @@ # ######################################################################### my $throttle_method = $o->get('throttle-method'); my @slaves; - if ( $throttle_method eq 'slavelag' ) { + if ( lc($throttle_method) eq 'slavelag' ) { if ( $o->get('check-slave-lag') ) { MKDEBUG && _d('Using --check-slave-lag DSN for throttle'); # OptionParser can't auto-copy DSN vals from a cmd line DSN @@ -5565,35 +5640,24 @@ foreach my $arg ( qw(o dbh q tp du ch args_for) ) { die "I need a $arg argument" unless $args{$arg}; } - my $o = $args{o}; my $dbh = $args{dbh}; + MKDEBUG && _d('Getting all schema objects'); my $si = new SchemaIterator( - Quoter => $args{q}, + dbh => $dbh, + OptionParser => $args{o}, + Quoter => $args{q}, ); - $si->set_filter($si->make_filter($o)); - - my $next_db = $si->get_db_itr(dbh => $dbh); - while ( my $db = $next_db->() ) { - my $next_tbl = $si->get_tbl_itr( - dbh => $dbh, - db => $db, - views => 0, + while ( my %schema_obj = $si->next_schema_object() ) { + my $final_o = get_final_opts( + %args, + %schema_obj, + ); + save_tbl_to_checksum( + %args, + %schema_obj, + final_o => $final_o, ); - while ( my $tbl = $next_tbl->() ) { - MKDEBUG && _d("Examining", $db, '.', $tbl); - my $final_o = get_final_opts( - %args, - db => $db, - tbl => $tbl, - ); - save_tbl_to_checksum( - %args, - db => $db, - tbl => $tbl, - final_o => $final_o, - ); - } } return; @@ -6778,29 +6842,36 @@ my $crc = 'NULL'; my $sta = 'NULL'; my $lag = 'NULL'; + + # Begin timing the checksum operation. my $beg = time(); # I'm a slave. Wait to catch up to the master. Calculate slave lag. - if ( !$is_master - && $final_o->get('wait') - && !$final_o->get('explain') ) - { - MKDEBUG && _d('Waiting to catch up to master', $args{dbh}->{h}, - ':', $args{dbh}->{P}); - $sta = $ms->wait_for_master( - $args{dbh}, $dbh, $final_o->get('wait'), 1, $tbl->{master_status}); - $sta = 'NULL' unless defined $sta; - } - if ( !$is_master - && $final_o->get('slave-lag') - && !$final_o->get('explain') ) - { - my $res = $ms->get_slave_status($dbh); - $lag = $res && defined $res->{seconds_behind_master} - ? $res->{seconds_behind_master} - : 'NULL'; + if ( !$is_master && !$final_o->get('explain') ) { + if ( $final_o->get('wait') ) { + MKDEBUG && _d('Waiting to catch up to master for --wait'); + my $result = $ms->wait_for_master( + master_status => $tbl->{master_status}, + slave_dbh => $dbh, + timeout => $final_o->get('wait'), + ); + $sta = $result && defined $result->{result} + ? $result->{result} + : 'NULL'; + } + + if ( $final_o->get('slave-lag') ) { + MKDEBUG && _d('Getting slave lag for --slave-lag'); + my $res = $ms->get_slave_status($dbh); + $lag = $res && defined $res->{seconds_behind_master} + ? $res->{seconds_behind_master} + : 'NULL'; + } } + # Time the checksum operation and the wait-for-master operation separately. + my $mid = time(); + # Check that table exists on slave. my $have_table = 1; if ( !$is_master || !$checksum_table_data ) { @@ -6815,9 +6886,6 @@ unless $have_table; } - # Time the checksum operation and the wait-for-master operation separately. - my $mid = time(); - if ( $have_table ) { # Do the checksum operation. if ( $checksum_table_data ) { @@ -7563,19 +7631,23 @@ =item TIME -The time the actual checksum and/or counting took. +How long it took to checksum the C, not including C time. +Total checksum time is C. =item WAIT -How long the checksum blocked before beginning. +How long the slave waited to catch up to its master before beginning to +checksum. C is always 0 for the master. See L<"--wait">. =item STAT -The return value of MASTER_POS_WAIT(). +The return value of MASTER_POS_WAIT(). C is always C for the +master. =item LAG How far the slave lags the master, as reported by SHOW SLAVE STATUS. +C is always C for the master. =back @@ -8851,6 +8923,6 @@ =head1 VERSION -This manual page documents Ver 1.2.22 Distrib 7486 $Revision: 7480 $. +This manual page documents Ver 1.2.23 Distrib 7540 $Revision: 7527 $. =cut diff -Nru maatkit-7486/bin/mk-table-sync maatkit-7540/bin/mk-table-sync --- maatkit-7486/bin/mk-table-sync 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-table-sync 2011-06-08 16:49:31.000000000 +0000 @@ -24,7 +24,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.31'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7476 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -10284,6 +10284,6 @@ =head1 VERSION -This manual page documents Ver 1.0.31 Distrib 7486 $Revision: 7476 $. +This manual page documents Ver 1.0.31 Distrib 7540 $Revision: 7476 $. =cut diff -Nru maatkit-7486/bin/mk-table-usage maatkit-7540/bin/mk-table-usage --- maatkit-7486/bin/mk-table-usage 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-table-usage 2011-06-08 16:49:31.000000000 +0000 @@ -20,9 +20,9 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '1.0.0'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $VERSION = '1.0.1'; +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # DSNParser package 7388 @@ -1403,7 +1403,7 @@ # ########################################################################### # ########################################################################### -# SlowLogParser package 7096 +# SlowLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SlowLogParser.pm @@ -1573,6 +1573,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # EVENT @@ -2715,7 +2719,7 @@ # ########################################################################### # ########################################################################### -# SQLParser package 7433 +# SQLParser package 7497 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SQLParser.pm @@ -2762,6 +2766,7 @@ sub new { my ( $class, %args ) = @_; my $self = { + %args, }; return bless $self, $class; } @@ -3563,6 +3568,22 @@ else { die "Invalid number of parts in $type reference: $ident"; } + + if ( $self->{SchemaQualifier} ) { + if ( $type eq 'column' && !$ident_struct{tbl} ) { + my $qcol = $self->{SchemaQualifier}->qualify_column( + column => $ident_struct{col}, + ); + $ident_struct{db} = $qcol->{db} if $qcol->{db}; + $ident_struct{tbl} = $qcol->{tbl} if $qcol->{tbl}; + } + elsif ( $type eq 'table' && !$ident_struct{db} ) { + my $db = $self->{SchemaQualifier}->get_database_for_table( + table => $ident_struct{tbl}, + ); + $ident_struct{db} = $db if $db; + } + } MKDEBUG && _d($type, "identifier struct:", Dumper(\%ident_struct)); return \%ident_struct; @@ -3598,6 +3619,12 @@ return 0; } +sub set_SchemaQualifier { + my ( $self, $sq ) = @_; + $self->{SchemaQualifier} = $sq; + return; +} + sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } @@ -3614,7 +3641,7 @@ # ########################################################################### # ########################################################################### -# TableUsage package 7458 +# TableUsage package 7498 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/TableUsage.pm @@ -3874,6 +3901,7 @@ where => $table->{join}->{where}, clause => 'JOIN condition', # just for debugging ); + MKDEBUG && _d("JOIN ON tables:", Dumper($on_tables)); foreach my $joined_table ( @{$on_tables->{joined_tables}} ) { $self->_change_context( tables => $tables, @@ -4039,8 +4067,9 @@ foreach my $cond ( @$where ) { MKDEBUG && _d("Condition:", Dumper($cond)); my @tables; # tables used in this condition - my $n_vals = 0; - my $is_constant = 0; + my $n_vals = 0; + my $is_constant = 0; + my $unknown_table = 0; ARG: foreach my $arg ( qw(left_arg right_arg) ) { if ( !defined $cond->{$arg} ) { @@ -4065,6 +4094,10 @@ else { MKDEBUG && _d("Condition column is not table-qualified and", "query has multiple tables; cannot determine its table"); + if ( $cond->{$arg} !~ m/\w+\(/ # not a function + && $cond->{$arg} !~ m/^[\d.]+$/) { # not a number + $unknown_table = 1; + } next ARG; } } @@ -4095,8 +4128,15 @@ } else { if ( @tables == 1 ) { - MKDEBUG && _d("Condition filters table", $tables[0]); - $filter_tables{$tables[0]} = undef; + if ( $unknown_table ) { + MKDEBUG && _d("Condition joins table", + $tables[0], "to column from unknown table"); + $join_tables{$tables[0]} = undef; + } + else { + MKDEBUG && _d("Condition filters table", $tables[0]); + $filter_tables{$tables[0]} = undef; + } } elsif ( @tables == 2 ) { MKDEBUG && _d("Condition joins tables", @@ -4732,7 +4772,7 @@ # ########################################################################### # ########################################################################### -# Pipeline package 7268 +# Pipeline package 7509 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/Pipeline.pm @@ -4814,6 +4854,8 @@ my $pipeline_data = $args{pipeline_data} || {}; $pipeline_data->{oktorun} = $oktorun; + my $stats = $args{stats}; # optional + MKDEBUG && _d("Pipeline starting at", time); my $instrument = $self->{instrument}; my $processes = $self->{procs}; @@ -4840,6 +4882,10 @@ if ( !$output ) { MKDEBUG && _d("Pipeline restarting early after", $self->{names}->[$procno]); + if ( $stats ) { + $stats->{"pipeline_restarted_after_" + .$self->{names}->[$procno]}++; + } last PIPELINE_PROCESS; } $procno++; @@ -4890,6 +4936,780 @@ # ########################################################################### # ########################################################################### +# Quoter package 6850 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/Quoter.pm +# trunk/common/t/Quoter.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### + +package Quoter; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +sub new { + my ( $class, %args ) = @_; + return bless {}, $class; +} + +sub quote { + my ( $self, @vals ) = @_; + foreach my $val ( @vals ) { + $val =~ s/`/``/g; + } + return join('.', map { '`' . $_ . '`' } @vals); +} + +sub quote_val { + my ( $self, $val ) = @_; + + return 'NULL' unless defined $val; # undef = NULL + return "''" if $val eq ''; # blank string = '' + return $val if $val =~ m/^0x[0-9a-fA-F]+$/; # hex data + + $val =~ s/(['\\])/\\$1/g; + return "'$val'"; +} + +sub split_unquote { + my ( $self, $db_tbl, $default_db ) = @_; + $db_tbl =~ s/`//g; + my ( $db, $tbl ) = split(/[.]/, $db_tbl); + if ( !$tbl ) { + $tbl = $db; + $db = $default_db; + } + return ($db, $tbl); +} + +sub literal_like { + my ( $self, $like ) = @_; + return unless $like; + $like =~ s/([%_])/\\$1/g; + return "'$like'"; +} + +sub join_quote { + my ( $self, $default_db, $db_tbl ) = @_; + return unless $db_tbl; + my ($db, $tbl) = split(/[.]/, $db_tbl); + if ( !$tbl ) { + $tbl = $db; + $db = $default_db; + } + $db = "`$db`" if $db && $db !~ m/^`/; + $tbl = "`$tbl`" if $tbl && $tbl !~ m/^`/; + return $db ? "$db.$tbl" : $tbl; +} + +1; + +# ########################################################################### +# End Quoter package +# ########################################################################### + +# ########################################################################### +# TableParser package 7156 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/TableParser.pm +# trunk/common/t/TableParser.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### + +package TableParser; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +sub new { + my ( $class, %args ) = @_; + my @required_args = qw(Quoter); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my $self = { %args }; + return bless $self, $class; +} + +sub parse { + my ( $self, $ddl, $opts ) = @_; + return unless $ddl; + if ( ref $ddl eq 'ARRAY' ) { + if ( lc $ddl->[0] eq 'table' ) { + $ddl = $ddl->[1]; + } + else { + return { + engine => 'VIEW', + }; + } + } + + if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) { + die "Cannot parse table definition; is ANSI quoting " + . "enabled or SQL_QUOTE_SHOW_CREATE disabled?"; + } + + my ($name) = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/; + (undef, $name) = $self->{Quoter}->split_unquote($name) if $name; + + $ddl =~ s/(`[^`]+`)/\L$1/g; + + my $engine = $self->get_engine($ddl); + + my @defs = $ddl =~ m/^(\s+`.*?),?$/gm; + my @cols = map { $_ =~ m/`([^`]+)`/ } @defs; + MKDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols)); + + my %def_for; + @def_for{@cols} = @defs; + + my (@nums, @null); + my (%type_for, %is_nullable, %is_numeric, %is_autoinc); + foreach my $col ( @cols ) { + my $def = $def_for{$col}; + my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/; + die "Can't determine column type for $def" unless $type; + $type_for{$col} = $type; + if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) { + push @nums, $col; + $is_numeric{$col} = 1; + } + if ( $def !~ m/NOT NULL/ ) { + push @null, $col; + $is_nullable{$col} = 1; + } + $is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0; + } + + my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable); + + my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/; + + return { + name => $name, + cols => \@cols, + col_posn => { map { $cols[$_] => $_ } 0..$#cols }, + is_col => { map { $_ => 1 } @cols }, + null_cols => \@null, + is_nullable => \%is_nullable, + is_autoinc => \%is_autoinc, + clustered_key => $clustered_key, + keys => $keys, + defs => \%def_for, + numeric_cols => \@nums, + is_numeric => \%is_numeric, + engine => $engine, + type_for => \%type_for, + charset => $charset, + }; +} + +sub sort_indexes { + my ( $self, $tbl ) = @_; + + my @indexes + = sort { + (($a ne 'PRIMARY') <=> ($b ne 'PRIMARY')) + || ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} ) + || ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} ) + || ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) ) + } + grep { + $tbl->{keys}->{$_}->{type} eq 'BTREE' + } + sort keys %{$tbl->{keys}}; + + MKDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes)); + return @indexes; +} + +sub find_best_index { + my ( $self, $tbl, $index ) = @_; + my $best; + if ( $index ) { + ($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}}; + } + if ( !$best ) { + if ( $index ) { + die "Index '$index' does not exist in table"; + } + else { + ($best) = $self->sort_indexes($tbl); + } + } + MKDEBUG && _d('Best index found is', $best); + return $best; +} + +sub find_possible_keys { + my ( $self, $dbh, $database, $table, $quoter, $where ) = @_; + return () unless $where; + my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table) + . ' WHERE ' . $where; + MKDEBUG && _d($sql); + my $expl = $dbh->selectrow_hashref($sql); + $expl = { map { lc($_) => $expl->{$_} } keys %$expl }; + if ( $expl->{possible_keys} ) { + MKDEBUG && _d('possible_keys =', $expl->{possible_keys}); + my @candidates = split(',', $expl->{possible_keys}); + my %possible = map { $_ => 1 } @candidates; + if ( $expl->{key} ) { + MKDEBUG && _d('MySQL chose', $expl->{key}); + unshift @candidates, grep { $possible{$_} } split(',', $expl->{key}); + MKDEBUG && _d('Before deduping:', join(', ', @candidates)); + my %seen; + @candidates = grep { !$seen{$_}++ } @candidates; + } + MKDEBUG && _d('Final list:', join(', ', @candidates)); + return @candidates; + } + else { + MKDEBUG && _d('No keys in possible_keys'); + return (); + } +} + +sub check_table { + my ( $self, %args ) = @_; + my @required_args = qw(dbh db tbl); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($dbh, $db, $tbl) = @args{@required_args}; + my $q = $self->{Quoter}; + my $db_tbl = $q->quote($db, $tbl); + MKDEBUG && _d('Checking', $db_tbl); + + my $sql = "SHOW TABLES FROM " . $q->quote($db) + . ' LIKE ' . $q->literal_like($tbl); + MKDEBUG && _d($sql); + my $row; + eval { + $row = $dbh->selectrow_arrayref($sql); + }; + if ( $EVAL_ERROR ) { + MKDEBUG && _d($EVAL_ERROR); + return 0; + } + if ( !$row->[0] || $row->[0] ne $tbl ) { + MKDEBUG && _d('Table does not exist'); + return 0; + } + + MKDEBUG && _d('Table exists; no privs to check'); + return 1 unless $args{all_privs}; + + $sql = "SHOW FULL COLUMNS FROM $db_tbl"; + MKDEBUG && _d($sql); + eval { + $row = $dbh->selectrow_hashref($sql); + }; + if ( $EVAL_ERROR ) { + MKDEBUG && _d($EVAL_ERROR); + return 0; + } + if ( !scalar keys %$row ) { + MKDEBUG && _d('Table has no columns:', Dumper($row)); + return 0; + } + my $privs = $row->{privileges} || $row->{Privileges}; + + $sql = "DELETE FROM $db_tbl LIMIT 0"; + MKDEBUG && _d($sql); + eval { + $dbh->do($sql); + }; + my $can_delete = $EVAL_ERROR ? 0 : 1; + + MKDEBUG && _d('User privs on', $db_tbl, ':', $privs, + ($can_delete ? 'delete' : '')); + + if ( !($privs =~ m/select/ && $privs =~ m/insert/ && $privs =~ m/update/ + && $can_delete) ) { + MKDEBUG && _d('User does not have all privs'); + return 0; + } + + MKDEBUG && _d('User has all privs'); + return 1; +} + +sub get_engine { + my ( $self, $ddl, $opts ) = @_; + my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; + MKDEBUG && _d('Storage engine:', $engine); + return $engine || undef; +} + +sub get_keys { + my ( $self, $ddl, $opts, $is_nullable ) = @_; + my $engine = $self->get_engine($ddl); + my $keys = {}; + my $clustered_key = undef; + + KEY: + foreach my $key ( $ddl =~ m/^ ((?:[A-Z]+ )?KEY .*)$/gm ) { + + next KEY if $key =~ m/FOREIGN/; + + my $key_ddl = $key; + MKDEBUG && _d('Parsed key:', $key_ddl); + + if ( $engine !~ m/MEMORY|HEAP/ ) { + $key =~ s/USING HASH/USING BTREE/; + } + + my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/; + my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/; + $type = $type || $special || 'BTREE'; + if ( $opts->{mysql_version} && $opts->{mysql_version} lt '004001000' + && $engine =~ m/HEAP|MEMORY/i ) + { + $type = 'HASH'; # MySQL pre-4.1 supports only HASH indexes on HEAP + } + + my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/; + my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0; + my @cols; + my @col_prefixes; + foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) { + my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/; + push @cols, $name; + push @col_prefixes, $prefix; + } + $name =~ s/`//g; + + MKDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols)); + + $keys->{$name} = { + name => $name, + type => $type, + colnames => $cols, + cols => \@cols, + col_prefixes => \@col_prefixes, + is_unique => $unique, + is_nullable => scalar(grep { $is_nullable->{$_} } @cols), + is_col => { map { $_ => 1 } @cols }, + ddl => $key_ddl, + }; + + if ( $engine =~ m/InnoDB/i && !$clustered_key ) { + my $this_key = $keys->{$name}; + if ( $this_key->{name} eq 'PRIMARY' ) { + $clustered_key = 'PRIMARY'; + } + elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) { + $clustered_key = $this_key->{name}; + } + MKDEBUG && $clustered_key && _d('This key is the clustered key'); + } + } + + return $keys, $clustered_key; +} + +sub get_fks { + my ( $self, $ddl, $opts ) = @_; + my $fks = {}; + + foreach my $fk ( + $ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg ) + { + my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/; + my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/; + my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/; + + if ( $parent !~ m/\./ && $opts->{database} ) { + $parent = "`$opts->{database}`.$parent"; + } + + $fks->{$name} = { + name => $name, + colnames => $cols, + cols => [ map { s/[ `]+//g; $_; } split(',', $cols) ], + parent_tbl => $parent, + parent_colnames=> $parent_cols, + parent_cols => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ], + ddl => $fk, + }; + } + + return $fks; +} + +sub remove_auto_increment { + my ( $self, $ddl ) = @_; + $ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m; + return $ddl; +} + +sub remove_secondary_indexes { + my ( $self, $ddl ) = @_; + my $sec_indexes_ddl; + my $tbl_struct = $self->parse($ddl); + + if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) { + my $clustered_key = $tbl_struct->{clustered_key}; + $clustered_key ||= ''; + + my @sec_indexes = map { + my $key_def = $_->{ddl}; + $key_def =~ s/([\(\)])/\\$1/g; + $ddl =~ s/\s+$key_def//i; + + my $key_ddl = "ADD $_->{ddl}"; + $key_ddl .= ',' unless $key_ddl =~ m/,$/; + $key_ddl; + } + grep { $_->{name} ne $clustered_key } + values %{$tbl_struct->{keys}}; + MKDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes)); + + if ( @sec_indexes ) { + $sec_indexes_ddl = join(' ', @sec_indexes); + $sec_indexes_ddl =~ s/,$//; + } + + $ddl =~ s/,(\n\) )/$1/s; + } + else { + MKDEBUG && _d('Not removing secondary indexes from', + $tbl_struct->{engine}, 'table'); + } + + return $ddl, $sec_indexes_ddl, $tbl_struct; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; + +# ########################################################################### +# End TableParser package +# ########################################################################### + +# ########################################################################### +# MysqldumpParser package 7500 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/MysqldumpParser.pm +# trunk/common/t/MysqldumpParser.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### +package MysqldumpParser; + +{ # package scope +use strict; +use warnings FATAL => 'all'; + +use English qw(-no_match_vars); +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +my $open_comment = qr{/\*!\d{5} }; + +sub new { + my ( $class, %args ) = @_; + my @required_args = qw(); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my $self = { + %args, + }; + return bless $self, $class; +} + +sub parse_create_tables { + my ( $self, %args ) = @_; + my @required_args = qw(file); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($file) = @args{@required_args}; + + MKDEBUG && _d('Parsing CREATE TABLE from', $file); + open my $fh, '<', $file + or die "Cannot open $file: $OS_ERROR"; + + local $INPUT_RECORD_SEPARATOR = ''; + + my %schema; + my $db = ''; + CHUNK: + while (defined(my $chunk = <$fh>)) { + MKDEBUG && _d('db:', $db, 'chunk:', $chunk); + if ($chunk =~ m/Database: (\S+)/) { + $db = $1; # XXX + $db =~ s/^`//; # strip leading ` + $db =~ s/`$//; # and trailing ` + MKDEBUG && _d('New db:', $db); + } + elsif ($chunk =~ m/CREATE TABLE/) { + MKDEBUG && _d('Chunk has CREATE TABLE'); + + if ($chunk =~ m/DROP VIEW IF EXISTS/) { + MKDEBUG && _d('Table is a VIEW, skipping'); + next CHUNK; + } + + my ($create_table) + = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms; + if ( !$create_table ) { + warn "Failed to parse CREATE TABLE from\n" . $chunk; + next CHUNK; + } + $create_table =~ s/ \*\/;\Z/;/; # remove end of version comment + + push @{$schema{$db}}, $create_table; + } + else { + MKDEBUG && _d('Chunk has other data, ignoring'); + } + } + + close $fh; + + return \%schema; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +} # package scope +1; + +# ########################################################################### +# End MysqldumpParser package +# ########################################################################### + +# ########################################################################### +# SchemaQualifier package 7499 +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the SVN repository at, +# trunk/common/SchemaQualifier.pm +# trunk/common/t/SchemaQualifier.t +# See http://code.google.com/p/maatkit/wiki/Developers for more information. +# ########################################################################### +package SchemaQualifier; + +{ # package scope +use strict; +use warnings FATAL => 'all'; + +use English qw(-no_match_vars); +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +sub new { + my ( $class, %args ) = @_; + my @required_args = qw(TableParser Quoter); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my $self = { + %args, + schema => {}, # db > tbl > col + duplicate_column_name => {}, + duplicate_table_name => {}, + }; + return bless $self, $class; +} + +sub schema { + my ( $self ) = @_; + return $self->{schema}; +} + +sub get_duplicate_column_names { + my ( $self ) = @_; + return keys %{$self->{duplicate_column_name}}; +} + +sub get_duplicate_table_names { + my ( $self ) = @_; + return keys %{$self->{duplicate_table_name}}; +} + +sub set_schema_from_mysqldump { + my ( $self, %args ) = @_; + my @required_args = qw(dump); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($dump) = @args{@required_args}; + + my $schema = $self->{schema}; + my $tp = $self->{TableParser}; + my %column_name; + my %table_name; + + DATABASE: + foreach my $db (keys %$dump) { + if ( !$db ) { + warn "Empty database from parsed mysqldump output"; + next DATABASE; + } + + TABLE: + foreach my $table_def ( @{$dump->{$db}} ) { + if ( !$table_def ) { + warn "Empty CREATE TABLE for database $db parsed from mysqldump output"; + next TABLE; + } + my $tbl_struct = $tp->parse($table_def); + $schema->{$db}->{$tbl_struct->{name}} = $tbl_struct->{is_col}; + + map { $column_name{$_}++ } @{$tbl_struct->{cols}}; + $table_name{$tbl_struct->{name}}++; + } + } + + map { $self->{duplicate_column_name}->{$_} = 1 } + grep { $column_name{$_} > 1 } + keys %column_name; + + map { $self->{duplicate_table_name}->{$_} = 1 } + grep { $table_name{$_} > 1 } + keys %table_name; + + MKDEBUG && _d('Schema:', Dumper($schema)); + return; +} + +sub qualify_column { + my ( $self, %args ) = @_; + my @required_args = qw(column); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($column) = @args{@required_args}; + + MKDEBUG && _d('Qualifying', $column); + my ($col, $tbl, $db) = reverse map { s/`//g; $_ } split /[.]/, $column; + MKDEBUG && _d('Column', $column, 'has db', $db, 'tbl', $tbl, 'col', $col); + + my %qcol = ( + db => $db, + tbl => $tbl, + col => $col, + ); + if ( !$qcol{tbl} ) { + @qcol{qw(db tbl)} = $self->get_table_for_column(column => $qcol{col}); + } + elsif ( !$qcol{db} ) { + $qcol{db} = $self->get_database_for_table(table => $qcol{tbl}); + } + else { + MKDEBUG && _d('Column is already database-table qualified'); + } + + return \%qcol; +} + +sub get_table_for_column { + my ( $self, %args ) = @_; + my @required_args = qw(column); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($col) = @args{@required_args}; + MKDEBUG && _d('Getting table for column', $col); + + if ( $self->{duplicate_column_name}->{$col} ) { + MKDEBUG && _d('Column name is duplicate, cannot qualify it'); + return; + } + + my $schema = $self->{schema}; + foreach my $db ( keys %{$schema} ) { + foreach my $tbl ( keys %{$schema->{$db}} ) { + if ( $schema->{$db}->{$tbl}->{$col} ) { + MKDEBUG && _d('Column is in database', $db, 'table', $tbl); + return $db, $tbl; + } + } + } + + MKDEBUG && _d('Failed to find column in any table'); + return; +} + +sub get_database_for_table { + my ( $self, %args ) = @_; + my @required_args = qw(table); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($tbl) = @args{@required_args}; + MKDEBUG && _d('Getting database for table', $tbl); + + if ( $self->{duplicate_table_name}->{$tbl} ) { + MKDEBUG && _d('Table name is duplicate, cannot qualify it'); + return; + } + + my $schema = $self->{schema}; + foreach my $db ( keys %{$schema} ) { + if ( $schema->{$db}->{$tbl} ) { + MKDEBUG && _d('Table is in database', $db); + return $db; + } + } + + MKDEBUG && _d('Failed to find table in any database'); + return; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +} # package scope +1; + +# ########################################################################### +# End SchemaQualifier package +# ########################################################################### + +# ########################################################################### # This is a combination of modules and programs in one -- a runnable module. # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. @@ -4958,6 +5778,27 @@ ); # ######################################################################## + # Parse the --create-table-definitions files. + # ######################################################################## + if ( my $files = $o->get('create-table-definitions') ) { + my $q = new Quoter(); + my $tp = new TableParser(Quoter => $q); + my $sq = new SchemaQualifier(TableParser => $tp, Quoter => $q); + + my $dump_parser = new MysqldumpParser(); + FILE: + foreach my $file ( @$files ) { + my $dump = $dump_parser->parse_create_tables(file => $file); + if ( !$dump || !keys %$dump ) { + warn "No CREATE TABLE statements were found in $file"; + next FILE; + } + $sq->set_schema_from_mysqldump(dump => $dump); + } + $sp->set_SchemaQualifier($sq); + } + + # ######################################################################## # Set up an array of callbacks. # ######################################################################## my $pipeline_data = { @@ -5676,6 +6517,19 @@ Continue parsing even if there is an error. +=item --create-table-definitions + +type: array + +Read C definitions from this list of comma-separated files. +If you cannot use L<"--explain-extended"> to fully qualify table and column +names, you can save the output of C to one or more files +and specify those files with this option. The tool will parse all +C definitions from the files and use this information to +qualify table and column names. If a column name is used in multiple tables, +or table name is used in multiple databases, these duplicates cannot be +qualified. + =item --daemonize Fork to the background and detach from the shell. POSIX @@ -6022,6 +6876,6 @@ =head1 VERSION -This manual page documents Ver 1.0.0 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.1 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-tcp-model maatkit-7540/bin/mk-tcp-model --- maatkit-7486/bin/mk-tcp-model 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-tcp-model 2011-06-08 16:49:31.000000000 +0000 @@ -20,9 +20,9 @@ use strict; use warnings FATAL => 'all'; -our $VERSION = '1.0.0'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7485 $ =~ m/(\d+)/g, 0)); +our $VERSION = '1.0.1'; +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7536 $ =~ m/(\d+)/g, 0)); # ########################################################################### # OptionParser package 7102 @@ -1560,7 +1560,7 @@ # ########################################################################### # ########################################################################### -# SimpleTCPDumpParser package 7472 +# SimpleTCPDumpParser package 7515 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SimpleTCPDumpParser.pm @@ -1608,30 +1608,58 @@ my $unix_timestamp = make_ts($ts) . $us; if ( $dst =~ m/\.$self->{port}$/o ) { - $sessions->{$src} = { - pos_in_log => $pos_in_log, - ts => $unix_timestamp, - id => $self->{requests}++, - }; + my $event; + if ( exists $sessions->{$src} && $sessions->{$src}->{status} eq 'R' ) { + $event = $self->make_event($src); + } + if ( exists $sessions->{$src} ) { + $sessions->{$src}->{ts} = $unix_timestamp; + } + else { + $sessions->{$src} ||= { + pos_in_log => $pos_in_log, + ts => $unix_timestamp, + ts0 => $unix_timestamp, + id => $self->{requests}++, + status => 'Q', + }; + } + return $event if $event; } elsif (defined (my $event = $sessions->{$dst}) ) { - my ( $src_host, $src_port ) = $dst =~ m/^(.*)\.(\d+)$/; - $event->{end} = $unix_timestamp; - $event->{host} = $src_host; - $event->{port} = $src_port; - $event->{arg} = undef; - MKDEBUG && _d('Properties of event:', Dumper($event)); - delete $sessions->{$dst}; - return $event; + $event->{status} = 'R', + $event->{end} ||= $unix_timestamp; + $event->{end1} = $unix_timestamp; } $pos_in_log = $tell->(); } # EVENT + foreach my $src ( keys %$sessions ) { + my $event = $self->make_event($src); + return $event if $event; + } + $args{oktorun}->(0) if $args{oktorun}; return; } +sub make_event { + my ( $self, $src ) = @_; + my $event = $self->{sessions}->{$src}; + delete $self->{sessions}->{$src}; + if ( $event->{status} eq 'R' ) { + my ( $src_host, $src_port ) = $src =~ m/^(.*)\.(\d+)$/; + $event->{host} = $src_host; + $event->{port} = $src_port; + $event->{arg} = undef; + delete $event->{status}; + MKDEBUG && _d('Properties of event:', Dumper($event)); + return $event; + } + return undef; +} + { my ($last, $result); sub make_ts { @@ -1660,7 +1688,7 @@ # ########################################################################### # ########################################################################### -# TCPRequestAggregator package 7473 +# TCPRequestAggregator package 7515 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/TCPRequestAggregator.pm @@ -1686,7 +1714,7 @@ my $self = { buffer => [], last_weighted_time => 0, - last_res_time => 0, + last_busy_time => 0, last_completions => 0, current_ts => 0, %args, @@ -1768,7 +1796,7 @@ MKDEBUG && _d("Timestamp doesn't belong to this interval"); if ( $self->{in_prg} ) { MKDEBUG && _d("Computing from", $self->{current_ts}, "to", $t_start); - $self->{res_time} += $t_start - $self->{current_ts}; + $self->{busy_time} += $t_start - $self->{current_ts}; $self->{weighted_time} += ($t_start - $self->{current_ts}) * $self->{in_prg}; } @@ -1790,7 +1818,7 @@ else { if ( $self->{in_prg} ) { MKDEBUG && _d("Computing from", $self->{current_ts}, "to", $timestamp); - $self->{res_time} += $timestamp - $self->{current_ts}; + $self->{busy_time} += $timestamp - $self->{current_ts}; $self->{weighted_time} += ($timestamp - $self->{current_ts}) * $self->{in_prg}; } $self->{current_ts} = $timestamp; @@ -1838,8 +1866,8 @@ my $e_throughput = sprintf( "%.6f", $e_arrivals / ( $t_end - $t_start ) ); my $e_completions = ( $self->{completions} - $self->{last_completions} ); - my $e_res_time - = sprintf( "%.6f", $self->{res_time} - $self->{last_res_time} ); + my $e_busy_time + = sprintf( "%.6f", $self->{busy_time} - $self->{last_busy_time} ); my $e_weighted_time = sprintf( "%.6f", $self->{weighted_time} - $self->{last_weighted_time} ); my $e_sum_time = sprintf("%.6f", $sum_times || 0); @@ -1852,7 +1880,7 @@ throughput => $e_throughput, arrivals => $e_arrivals, completions => $e_completions, - res_time => $e_res_time , + busy_time => $e_busy_time, weighted_time => $e_weighted_time, sum_time => $e_sum_time, variance_mean => $e_variance_mean, @@ -1864,7 +1892,7 @@ $self->{t_start} = $t_end; # Not current_timestamp! $self->{current_ts} = $t_end; # Next iteration will begin at boundary $self->{last_weighted_time} = $self->{weighted_time}; - $self->{last_res_time} = $self->{res_time}; + $self->{last_busy_time} = $self->{busy_time}; $self->{last_completions} = $self->{completions}; $self->{response_times} = []; @@ -1990,14 +2018,15 @@ # ##################################################################### # This is the main loop over the events in the input file. # ##################################################################### + my ($ts, $end) = @{$o->get('start-end')}; EVENT: while ( $event = $get_event->() ) { if ( $o->get('type') eq 'tcpdump' ) { printf "%6d %.6f %.6f %9.6f %s:%s\n", $event->{id}, - $event->{ts}, - $event->{end}, - $event->{end} - $event->{ts}, + $event->{$ts}, + $event->{$end}, + $event->{$end} - $event->{$ts}, $event->{host}, $event->{port}; } @@ -2091,6 +2120,14 @@ mk-tcp-model --type=requests --run-time=10 sorted.txt > sliced.txt +Transform the result for modeling with Aspersa's usl tool, discarding the first +and last line of each file if you specify multiple files (the first and last +line are normally incomplete observation periods and are aberrant): + + for f in sliced.txt; do + tail -n +2 "$f" | head -n -1 | awk '{print $2, $3, $7/$4}' + done > usl-input.txt + =head1 RISKS The following section is included to inform users about the potential risks, @@ -2115,26 +2152,29 @@ This tool recognizes requests and responses in a TCP stream, and extracts the "conversations". You can use it to capture the response times of individual -queries to a database, for example. It expects the input to be in the following -format, which should result from the sample shown in the SYNOPSIS: +queries to a database, for example. It expects the TCP input to be in the +following format, which should result from the sample shown in the SYNOPSIS: IP > : The tool watches for "incoming" packets to the port you specify with the L<"--watch-server"> option. This begins a request. If multiple inbound packets -follow each other, then the last inbound packet seen determines the time at -which the request is assumed to begin. This is logical if one assumes that a -server must receive the whole SQL statement before beginning execution, for -example. - -When the first outbound packet is seen, that concludes the request. We might -see an inbound packet, but never see a response. This can happen when the -kernel drops packets, for example. As a result, we never print a request until -we see the response to it, which means that the output is in completion order, -not arrival order. This can be useful for some kinds of analysis, but others -require processing in arrival order. Therefore, the second type of processing -this tool can do requires that you sort the output from the first stage and -supply it as input. +follow each other, then by default the last inbound packet seen determines the +time at which the request is assumed to begin. This is logical if one assumes +that a server must receive the whole SQL statement before beginning execution, +for example. + +When the first outbound packet is seen, the server is considered to have +responded to the request. The tool might see an inbound packet, but never see a +response. This can happen when the kernel drops packets, for example. As a +result, the tool never prints a request unless it sees the response to it. +However, the tool actually does not print any request until it sees the "last" +outbound packet. It determines this by waiting for either another inbound +packet, or EOF, and then considers the previous inbound/outbound pair to be +complete. As a result, the tool prints requests in a relatively random order. +Most types of analysis require processing in either arrival or completion order. +Therefore, the second type of processing this tool can do requires that you sort +the output from the first stage and supply it as input. The second type of processing is selected with the L<"--type"> option set to "requests". In this mode, the tool reads a group of requests and aggregates @@ -2145,10 +2185,16 @@ In the default mode (parsing tcpdump output), requests are printed out one per line, in the following format: - + + +The ID is an incrementing number, assigned in arrival order in the original TCP +traffic. The start and end timestamps, and the elapsed time, can be customized +with the L<"--start-end"> option. In "--type=requests" mode, the tool prints out one line per time interval as -defined by L<"--run-time">, with the following columns: +defined by L<"--run-time">, with the following columns: ts, concurrency, +throughput, arrivals, completions, busy_time, weighted_time, sum_time, +variance_mean, quantile_time, obs_time. A detailed explanation follows: =over @@ -2172,7 +2218,7 @@ The number of completions during the interval. -=item res_time +=item busy_time The total amount of time during which at least one request was resident in the server during the interval. @@ -2245,6 +2291,26 @@ The size of the aggregation interval in seconds when L<"--type"> is "requests" (default 1). Fractional values are permitted. +=item --start-end + +type: Array; default: ts,end + +Define how the arrival and completion timestamps of a query, and thus its +response time (elapsed time) are computed. Recall that there may be multiple +inbound and outbound packets per request and response, and refer to the +following ASCII diagram. Suppose that a client sends a series of three inbound +(I) packets to the server, whch computes the result and then sends two outbound +(O) packets back: + + I I I ..................... O O + |<---->|<---response time----->|<-->| + ts0 ts end end1 + +By default, the query is considered to arrive at time ts, and complete at time +end. However, this might not be what you want. Perhaps you do not want to +consider the query to have completed until time end1. You can accomplish this +by setting this option to C. + =item --type type: string @@ -2373,6 +2439,6 @@ =head1 VERSION -This manual page documents Ver 1.0.0 Distrib 7486 $Revision: 7485 $. +This manual page documents Ver 1.0.1 Distrib 7540 $Revision: 7536 $. =cut diff -Nru maatkit-7486/bin/mk-upgrade maatkit-7540/bin/mk-upgrade --- maatkit-7486/bin/mk-upgrade 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-upgrade 2011-06-08 16:49:31.000000000 +0000 @@ -21,8 +21,8 @@ use warnings FATAL => 'all'; our $VERSION = '0.9.8'; -our $DISTRIB = '7486'; -our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); +our $DISTRIB = '7540'; +our $SVN_REV = sprintf("%d", (q$Revision: 7531 $ =~ m/(\d+)/g, 0)); # ########################################################################### # DSNParser package 7388 @@ -2438,7 +2438,7 @@ # ########################################################################### # ########################################################################### -# SlowLogParser package 7096 +# SlowLogParser package 7522 # This package is a copy without comments from the original. The original # with comments and its test file can be found in the SVN repository at, # trunk/common/SlowLogParser.pm @@ -2608,6 +2608,10 @@ MKDEBUG && _d('Properties of event:', Dumper(\@properties)); my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } return $event; } # EVENT @@ -11815,6 +11819,6 @@ =head1 VERSION -This manual page documents Ver 0.9.8 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 0.9.8 Distrib 7540 $Revision: 7531 $. =cut diff -Nru maatkit-7486/bin/mk-variable-advisor maatkit-7540/bin/mk-variable-advisor --- maatkit-7486/bin/mk-variable-advisor 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-variable-advisor 2011-06-08 16:49:31.000000000 +0000 @@ -21,7 +21,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.2'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -3895,6 +3895,6 @@ =head1 VERSION -This manual page documents Ver 1.0.2 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.2 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/bin/mk-visual-explain maatkit-7540/bin/mk-visual-explain --- maatkit-7486/bin/mk-visual-explain 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/bin/mk-visual-explain 2011-06-08 16:49:31.000000000 +0000 @@ -23,7 +23,7 @@ use warnings FATAL => 'all'; our $VERSION = '1.0.22'; -our $DISTRIB = '7486'; +our $DISTRIB = '7540'; our $SVN_REV = sprintf("%d", (q$Revision: 7477 $ =~ m/(\d+)/g, 0)); # ########################################################################### @@ -3092,6 +3092,6 @@ =head1 VERSION -This manual page documents Ver 1.0.22 Distrib 7486 $Revision: 7477 $. +This manual page documents Ver 1.0.22 Distrib 7540 $Revision: 7477 $. =cut diff -Nru maatkit-7486/Changelog maatkit-7540/Changelog --- maatkit-7486/Changelog 2011-05-05 19:59:28.000000000 +0000 +++ maatkit-7540/Changelog 2011-06-08 16:49:29.000000000 +0000 @@ -788,6 +788,15 @@ Changelog for mk-heartbeat: +2011-06-08: version 1.0.23 + + * Removed --quiet option. + * Added --master-server-id option (issue 1292). + * Added --print-master-server-id option (issue 1292). + * Added --[no]insert-heartbeat-row option (issue 1292). + * Changed default heartbeat table structure (issue 1292). + * The tool crashed if only --stop was specified. + 2010-06-08: version 1.0.22 * Connections did not preserve server SQL modes (issue 801). @@ -1564,6 +1573,11 @@ Changelog for mk-query-digest: +2011-06-08: version 0.9.29 + + * --statistics did not print if no events were processed. + * Changed --statistics report format (issue 1320). + 2011-04-04: version 0.9.28 * Minimum 2-column query review history table did not work (issue 1265). @@ -2759,6 +2773,11 @@ Changelog for mk-table-checksum and mk-checksum-filter: +2011-06-08: version 1.2.23 + + * --wait did not work (issue 1319). + * Invalid --throttle-method values did not cause an error (issue 1309). + 2011-05-05: version 1.2.22 * --recurse did not work (issue 1039). @@ -3534,12 +3553,20 @@ Changelog for mk-table-usage: +2011-06-08: version 1.0.1 + + * Added --create-table-definitions option (issue 1316). + 2011-05-05: version 1.0.0 * Initial release. Changelog for mk-tcp-model: +2011-06-08: version 1.0.1 + + * Added --start-end option. + 2011-05-05: version 1.0.0 * Initial release. diff -Nru maatkit-7486/debian/changelog maatkit-7540/debian/changelog --- maatkit-7486/debian/changelog 2011-05-06 16:14:22.000000000 +0000 +++ maatkit-7540/debian/changelog 2011-06-09 00:14:39.000000000 +0000 @@ -1,3 +1,9 @@ +maatkit (7540-1) unstable; urgency=low + + * New upstream release (7540). (Closes: #629826) + + -- Dario Minnucci Thu, 09 Jun 2011 02:13:03 +0200 + maatkit (7486-1) unstable; urgency=low * New upstream release (7486). (Closes: #625873) diff -Nru maatkit-7486/maatkit.pod maatkit-7540/maatkit.pod --- maatkit-7486/maatkit.pod 2011-05-05 19:59:29.000000000 +0000 +++ maatkit-7540/maatkit.pod 2011-06-08 16:49:30.000000000 +0000 @@ -1,6 +1,6 @@ package maatkit; -our $VERSION = '7486'; +our $VERSION = '7540'; 1; @@ -19,7 +19,7 @@ The following tools are included: - $Revision: 7486 $ + $Revision: 7540 $ mk-archiver 1.0.27 mk-config-diff 1.0.0 mk-deadlock-logger 1.0.21 @@ -27,7 +27,7 @@ mk-error-log 1.0.3 mk-fifo-split 1.0.7 mk-find 0.9.23 -mk-heartbeat 1.0.22 +mk-heartbeat 1.0.23 mk-index-usage 0.9.4 mk-kill 0.9.10 mk-loadavg 0.9.7 @@ -36,7 +36,7 @@ mk-parallel-restore 1.0.24 mk-purge-logs 0.9.0 mk-query-advisor 1.0.4 -mk-query-digest 0.9.28 +mk-query-digest 0.9.29 mk-query-profiler 1.1.22 mk-show-grants 1.0.23 mk-slave-delay 1.0.23 @@ -44,10 +44,10 @@ mk-slave-move 0.9.12 mk-slave-prefetch 1.0.21 mk-slave-restart 1.0.22 -mk-table-checksum 1.2.22 +mk-table-checksum 1.2.23 mk-table-sync 1.0.31 -mk-table-usage 1.0.0 -mk-tcp-model 1.0.0 +mk-table-usage 1.0.1 +mk-tcp-model 1.0.1 mk-upgrade 0.9.8 mk-variable-advisor 1.0.2 mk-visual-explain 1.0.22 @@ -444,6 +444,6 @@ =head1 VERSION -This manual page documents Distrib 7486 $Revision: 534 $. +This manual page documents Distrib 7540 $Revision: 534 $. =cut diff -Nru maatkit-7486/maatkit.spec maatkit-7540/maatkit.spec --- maatkit-7486/maatkit.spec 2011-05-05 19:59:29.000000000 +0000 +++ maatkit-7540/maatkit.spec 2011-06-08 16:49:30.000000000 +0000 @@ -1,5 +1,5 @@ Name: maatkit -Version: 7486 +Version: 7540 Release: 1%{?dist} Summary: Essential command-line utilities for MySQL diff -Nru maatkit-7486/MANIFEST maatkit-7540/MANIFEST --- maatkit-7486/MANIFEST 2011-05-05 19:59:30.000000000 +0000 +++ maatkit-7540/MANIFEST 2011-06-08 16:49:31.000000000 +0000 @@ -1,46 +1,46 @@ -Changelog -udf/fnv_udf.cc +Makefile.PL udf/murmur_udf.cc -bin/mk-table-checksum -bin/mk-find -bin/mk-query-digest -bin/mk-kill -bin/mk-duplicate-key-checker -bin/mk-merge-mqd-results -bin/mk-purge-logs +udf/fnv_udf.cc bin/mk-config-diff +bin/mk-log-player +bin/mk-table-sync +bin/mk-parallel-restore +bin/mk-variable-advisor +bin/mk-deadlock-logger +bin/mk-index-usage +bin/mk-query-profiler +bin/mk-slave-move +bin/mk-visual-explain +bin/mk-kill +bin/mk-checksum-filter +bin/mk-table-usage +bin/mk-upgrade +bin/mk-fifo-split +bin/mk-query-advisor +bin/mk-slave-find +bin/mk-parallel-dump bin/mk-archiver bin/mk-loadavg -bin/mk-profile-compact -bin/mk-fifo-split -bin/mk-variable-advisor bin/mk-slave-prefetch -bin/mk-slave-find +bin/mk-heartbeat +bin/mk-error-log +bin/mk-show-grants bin/mk-slave-delay -bin/mk-log-player +bin/mk-table-checksum +bin/mk-profile-compact +bin/mk-duplicate-key-checker +bin/mk-find bin/mk-tcp-model -bin/mk-index-usage +bin/mk-merge-mqd-results +bin/mk-query-digest bin/mk-slave-restart -bin/mk-error-log -bin/mk-query-advisor -bin/mk-show-grants -bin/mk-upgrade -bin/mk-heartbeat -bin/mk-deadlock-logger -bin/mk-parallel-dump -bin/mk-parallel-restore -bin/mk-checksum-filter -bin/mk-table-usage -bin/mk-table-sync -bin/mk-query-profiler -bin/mk-slave-move -bin/mk-visual-explain -init/maatkit -maatkit.pod +bin/mk-purge-logs MANIFEST +Changelog +maatkit.spec README -Makefile.PL COPYING INSTALL -maatkit.spec +init/maatkit +maatkit.pod MANIFEST