diff --git a/scripts/zmaudit.pl.z b/scripts/zmaudit.pl.z new file mode 100755 index 000000000..e9a123432 --- /dev/null +++ b/scripts/zmaudit.pl.z @@ -0,0 +1,235 @@ +#!/usr/bin/perl -w +# +# This script checks for consistency between the event filesystem and +# the database. If events are found in one and not the other they are +# deleted (optionally). Additionally any monitor event directories that +# do not correspond to a database monitor are similarly disposed of. +# However monitors in the database that don't have a directory are left +# alone as this is valid if they are newly created and have no events +# yet. +# + +use strict; +use DBI; +use Getopt::Long; +use Term::ReadKey; + +$| = 1; + +my $report = 0; +my $yes = 0; +my $delay = 0; + +sub usage +{ + print( " +Usage: zme.pl [-r,-report|-y,-yes] [-d ,-delay=] +Parameters are :- +-r, --report - Just report don't actually do anything +-y, --yes - Just do all actions without confirmation +-d , --delay= - how long to delay between each delete action, default 0 +"); + exit( -1 ); +} + +sub confirm +{ + my $yesno = $yes?1:0; + if ( $report ) + { + print( "\n" ); + } + elsif ( $yes ) + { + print( ", deleting\n" ); + } + else + { + print( ", delete y/n: " ); + my $char = ReadKey( 0 ); + #print( "C:'".ord($char)."'\n" ); + if ( $char eq 'q' ) + { + print( "\n" ); + exit( 0 ); + } + if ( ord($char) == 10 ) + { + $char = 'y'; + } + else + { + print( "$char\n" ); + } + if ( $char eq "a" ) + { + $yes = 1; + return( 1 ); + } + $yesno = ( $char =~ /[yY]/ ); + } + return( $yesno ); +} + +if ( !GetOptions( 'report'=>\$report, 'yes'=>\$yes, 'delay=i'=>\$delay ) ) +{ + usage(); +} + +if ( $report && $yes ) +{ + print( STDERR "Error, only one of --report and --yes may be specified\n" ); + usage(); +} +if ( $report && $delay ) +{ + print( STDERR "Error, using --report and --delay is meaningless (and bogus)\n" ); + usage(); +} + +ReadMode( 'cbreak' ); + +my $event_path = "/data/zm/events"; + +chdir( $event_path ); + +my $fs_monitors; +foreach my $monitor ( <*> ) +{ + print( "Found filesystem monitor '$monitor'" ); + my $fs_events = $fs_monitors->{$monitor} = {}; + if ( 1 ) { + chdir( $monitor ); + foreach my $event ( <*> ) + { + $fs_events->{$event} = !undef; + } + chdir( '..' ); + } + print( ", got ".int(keys(%$fs_events))." events\n" ); +} + +my $database = "zm"; +my $username = "zmadmin"; +my $password = "zmadminzm"; + +my $dbh = DBI->connect( "DBI:mysql:$database", $username, $password ); + +my $db_monitors; +my $sql = "select * from Monitors order by Name"; +my $sth = $dbh->prepare_cached( $sql ) or die( "Can't prepare '$sql': ".$dbh->errstr() ); +my $sql2 = "select * from Events where MonitorId = ? order by Id"; +my $sth2 = $dbh->prepare_cached( $sql2 ) or die( "Can't prepare '$sql2': ".$dbh->errstr() ); +my $res = $sth->execute() or die( "Can't execute: ".$sth->errstr() ); +while( my $monitor = $sth->fetchrow_hashref() ) +{ + print( "Found database monitor '$monitor->{Name}'" ); + my $db_events = $db_monitors->{$monitor->{Name}} = {}; + my $res = $sth2->execute( $monitor->{Id} ) or die( "Can't execute: ".$sth2->errstr() ); + while ( my $event = $sth2->fetchrow_hashref() ) + { + $db_events->{$event->{Id}} = !undef; + } + print( ", got ".int(keys(%$db_events))." events\n" ); + $sth2->finish(); +} +$sth->finish(); + +while ( my ( $fs_monitor, $fs_events ) = each(%$fs_monitors) ) +{ + if ( my $db_events = $db_monitors->{$fs_monitor} ) + { + if ( $fs_events ) + { + while ( my ( $fs_event, $val ) = each(%$fs_events ) ) + { + if ( !$db_events->{$fs_event} ) + { + print( "Filesystem event '$fs_monitor/$fs_event' does not exist in database" ); + if ( confirm() ) + { + qx( rm -rf $event_path/$fs_monitor/$fs_event ); + sleep( $delay ) if ( $delay ); + } + } + } + } + } + else + { + print( "Filesystem monitor '$fs_monitor' does not exist in database" ); + if ( confirm() ) + { + qx( rm -rf $event_path/$fs_monitor ); + sleep( $delay ) if ( $delay ); + } + } +} + +my $sql3 = "delete from Monitors where Id = ?"; +my $sth3 = $dbh->prepare_cached( $sql3 ) or die( "Can't prepare '$sql3': ".$dbh->errstr() ); +my $sql4 = "delete from Events where Id = ?"; +my $sth4 = $dbh->prepare_cached( $sql4 ) or die( "Can't prepare '$sql4': ".$dbh->errstr() ); +my $sql5 = "delete from Frames where EventId = ?"; +my $sth5 = $dbh->prepare_cached( $sql5 ) or die( "Can't prepare '$sql5': ".$dbh->errstr() ); +my $sql6 = "delete from Stats where EventId = ?"; +my $sth6 = $dbh->prepare_cached( $sql6 ) or die( "Can't prepare '$sql6': ".$dbh->errstr() ); +while ( my ( $db_monitor, $db_events ) = each(%$db_monitors) ) +{ + if ( my $fs_events = $fs_monitors->{$db_monitor} ) + { + if ( $db_events ) + { + while ( my ( $db_event, $val ) = each(%$db_events ) ) + { + if ( !$fs_events->{$db_event} ) + { + print( "Database event '$db_monitor/$db_event' does not exist in filesystem" ); + if ( confirm() ) + { + my $res = $sth4->execute( $db_event ) or die( "Can't execute: ".$sth4->errstr() ); + $res = $sth5->execute( $db_event ) or die( "Can't execute: ".$sth5->errstr() ); + $res = $sth6->execute( $db_event ) or die( "Can't execute: ".$sth5->errstr() ); + sleep( $delay ) if ( $delay ); + } + } + } + } + } + else + { + print( "Database monitor '$db_monitor' does not exist in filesystem" ); + if ( confirm() ) + { + # We don't actually do this in case it's new + #my $res = $sth3->execute( $db_monitor ) or die( "Can't execute: ".$sth3->errstr() ); + #sleep( $delay ) if ( $delay ); + } + } +} + +my $sql7 = "select distinct EventId from Frames left join Events on Frames.EventId = Events.Id where isnull(Events.Id)"; +my $sth7 = $dbh->prepare_cached( $sql7 ) or die( "Can't prepare '$sql7': ".$dbh->errstr() ); +$res = $sth7->execute() or die( "Can't execute: ".$sth7->errstr() ); +while( my $frame = $sth7->fetchrow_hashref() ) +{ + print( "Found orphaned frame records for event '$frame->{EventId}'" ); + if ( confirm() ) + { + $res = $sth7->execute( $frame->{EventId} ) or die( "Can't execute: ".$sth7->errstr() ); + } +} + +my $sql8 = "select distinct EventId from Stats left join Events on Stats.EventId = Events.Id where isnull(Events.Id)"; +my $sth8 = $dbh->prepare_cached( $sql8 ) or die( "Can't prepare '$sql8': ".$dbh->errstr() ); +$res = $sth8->execute() or die( "Can't execute: ".$sth8->errstr() ); +while( my $stat = $sth8->fetchrow_hashref() ) +{ + print( "Found orphaned statistic records for event '$stat->{EventId}'" ); + if ( confirm() ) + { + $res = $sth8->execute( $stat->{EventId} ) or die( "Can't execute: ".$sth8->errstr() ); + } +} + +ReadMode( 'restore' ); diff --git a/scripts/zmfilter.pl.z b/scripts/zmfilter.pl.z new file mode 100755 index 000000000..d4673922b --- /dev/null +++ b/scripts/zmfilter.pl.z @@ -0,0 +1,321 @@ +#!/usr/bin/perl -w +# +# This script continuously monitors the recorded events for the given +# monitor and applies any filters which would delete and/or upload +# matching events +# +use strict; + +# ========================================================================== +# +# These are the elements you need to edit to suit your installation +# +# ========================================================================== +use constant DB_NAME => "zm"; +use constant DB_USER => "zmadmin"; +use constant DB_PASS => "zmadminzm"; +use constant SHM_KEY => 0x7a6d2000; +use constant EVENT_PATH => "/data/zm/events"; +use constant FILTER_RELOAD_DELAY => 300; # How often filters are reloaded + +# Comment these out if you don't have them and don't want to upload +# or don't want to use that format +use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); +use Archive::Tar; +use Net::FTP; + +use constant ARCH_FORMAT => 'tar'; # This can be 'tar' or 'zip' +use constant ARCH_COMPRESS => '0'; # Whether to compress archive files + +use constant FTP_HOST => 'raq182.uk2net.com'; +use constant FTP_USER => 'admin'; +use constant FTP_PASS => 'w712jhw'; +use constant FTP_LOC_DIR => '/tmp/'; +use constant FTP_REM_DIR => 'transfer/'; +use constant FTP_TIMEOUT => 120; +use constant FTP_PASSIVE => 1; +use constant FTP_DEBUG => 0; + +# ========================================================================== +# +# You shouldn't need to change anything from here downwards +# +# ========================================================================== + +use DBI; +use IPC::SysV qw/IPC_CREAT/; +use Date::Manip; +use Data::Dumper; +use Getopt::Long; + +$| = 1; + +my $monitor = 0; +my $delay = 30; +my $event_id = 0; + +sub Usage +{ + print( " +Usage: zmf.pl (-m ,-monitor=) [-e ,--event=] [-d ,--delay=] +Parameters are :- +-m , --monitor= - Indicate which monitor id to run for +-e, --event= - What event to start scanning from +-d, --delay= - How long to delay between each check, default 30 +"); + exit( -1 ); +} + +# +# More or less replicates the equivalent PHP function +# +sub strtotime +{ + my $date_str = shift; + return( UnixDate( $date_str, '%s' ) ); +} + +# +# More or less replicates the equivalent PHP function +# +sub str_repeat +{ + my $string = shift; + my $count = shift; + return( ${string}x${count} ); +} + +if ( !GetOptions( 'monitor=i'=>\$monitor, 'event=i'=>\$event_id, 'delay=i'=>\$delay ) ) +{ + Usage(); +} + +if ( !$monitor ) +{ + print( STDERR "Please give a valid monitor id\n" ); + Usage(); +} + +chdir( EVENT_PATH ); + +my $dbh = DBI->connect( "DBI:mysql:".DB_NAME, DB_USER, DB_PASS ); + +my $last_event_id; + +if ( !$event_id ) +{ + my $sql = "select max(Id) as MaxEventId from Events where MonitorId = '$monitor'"; + my $sth = $dbh->prepare_cached( $sql ) or die( "Can't prepare '$sql': ".$dbh->errstr() ); + my $res = $sth->execute() or die( "Can't execute: ".$sth->errstr() ); + my $row = $sth->fetchrow_hashref(); + $event_id = $row->{MaxEventId}; + $sth->finish(); +} +print( "Scanning for events since id $event_id\n" ); + +my $filters = GetFilters( $monitor ); +my $last_action = 0; + +my $size = 16; # We only need the first 16 bytes really for the last event count +my $key = SHM_KEY|$monitor; +my $shmid = shmget( $key, $size, 0 ) || die( "Can't get shared memory id: $!" ); + +while( 1 ) +{ + shmread( $shmid, $last_event_id, 12, 4 ) || die( "Can't read from shared memory: $!" ); + $last_event_id = unpack( "l", $last_event_id ); + + print( "Last event generated is $last_event_id\n" ); + if ( $last_event_id > $event_id ) + { + # Got new event to check + if ( (time() - $last_action) > FILTER_RELOAD_DELAY ) + { + print( "Reloading filters\n" ); + $last_action = time(); + $filters = GetFilters( $monitor ); + + foreach my $filter ( @$filters ) + { + print( "Checking filter '$filter->{Name}'".($filter->{AutoDelete}?", delete":"").($filter->{AutoUpload}?", upload":"")."\n" ); + my $sql = $filter->{Sql}; + my $sth = $dbh->prepare_cached( $sql ) or die( "Can't prepare '$sql': ".$dbh->errstr() ); + my $res = $sth->execute( $event_id ) or die( "Can't execute: ".$sth->errstr() ); + + while( my $event = $sth->fetchrow_hashref() ) + { + print( "Checking event $event->{Id}\n" ); + if ( $filter->{AutoUpload} ) + { + my $arch_file = FTP_LOC_DIR.$filter->{MonitorName}.'-'.$event->{Id}; + my $arch_image_path = "$filter->{MonitorName}/$event->{Id}/*.jpg"; + my $arch_error; + + if ( ARCH_FORMAT eq "zip" ) + { + $arch_file .= '.zip'; + my $zip = Archive::Zip->new(); + print( "Creating upload file '$arch_file'\n" ); + + my $status = AZ_OK; + foreach my $image_file ( <*$arch_image_path> ) + { + print( "Adding $image_file\n" ); + my $member = $zip->addFile( $image_file ); + last unless ( $member ); + $member->desiredCompressionMethod( ARCH_COMPRESS?COMPRESSION_DEFLATED:COMPRESSION_STORED ); + } + $status = $zip->writeToFileNamed( $arch_file ); + + if ( $arch_error = ($status != AZ_OK) ) + { + print( "Zip error: $status\n " ); + } + } + elsif ( ARCH_FORMAT eq "tar" ) + { + if ( ARCH_COMPRESS ) + { + $arch_file .= '.tar.gz'; + } + else + { + $arch_file .= '.tar'; + } + print( "Creating upload file '$arch_file'\n" ); + + if ( $arch_error = !Archive::Tar->create_archive( $arch_file, ARCH_COMPRESS, <*$arch_image_path> ) ) + { + print( "Tar error: ".Archive::Tar->error()."\n " ); + } + } + + + if ( !$arch_error ) + { + print( "Uploading to ".FTP_HOST."\n" ); + my $ftp = Net::FTP->new( FTP_HOST, Timeout=>FTP_TIMEOUT, Passive=>FTP_PASSIVE, Debug=>FTP_DEBUG ); + $ftp->login( FTP_USER, FTP_PASS ) or die( "FTP - Can't login" ); + $ftp->binary() or die( "FTP - Can't go binary" ); + $ftp->cwd( FTP_REM_DIR ) or die( "FTP - Can't cwd" ); + $ftp->put( $arch_file ) or die( "FTP - Can't upload '$arch_file'" ); + $ftp->quit() or warn( "FTP - Can't quit" ); + unlink( $arch_file ); + } + } + if ( $filter->{AutoDelete} ) + { + print( "Deleting event\n" ); + # Do it individually to avoid locking up the table for new events + my $sql = "delete from Events where Id = ?"; + my $sth = $dbh->prepare_cached( $sql ) or die( "Can't prepare '$sql': ".$dbh->errstr() ); + #my $res = $sth->execute( $event->{Id} ) or die( "Can't execute: ".$sth->errstr() ); + + # We could now delete from the Frames and Stats table and the files themselves, + # but we can let the database checker sort that out for us instead. + } + } + $sth->finish(); + } + } + $event_id = $last_event_id; + } + else + { + print( "Sleeping for $delay seconds\n" ); + sleep( $delay ); + } +} + +sub GetFilters +{ + my $monitor = shift; + + my @filters; + my $sql = "select Filters.*,Monitors.Name as MonitorName from Filters inner join Monitors on Filters.MonitorId = Monitors.Id where AutoDelete = 1 or AutoUpload = 1 order by Name"; + my $sth = $dbh->prepare_cached( $sql ) or die( "Can't prepare '$sql': ".$dbh->errstr() ); + my $res = $sth->execute() or die( "Can't execute: ".$sth->errstr() ); + FILTER: while( my $filter_data = $sth->fetchrow_hashref() ) + { + print( "Found filter '$filter_data->{Name}'\n" ); + my %filter_terms; + foreach my $filter_parm ( split( '&', $filter_data->{Query} ) ) + { + my( $key, $value ) = split( '=', $filter_parm, 2 ); + if ( $key ) + { + $filter_terms{$key} = $value; + } + } + #print( Dumper( %filter_terms ) ); + my $sql = "select E.Id, E.Name,unix_timestamp(E.StartTime) as Time,E.Length,E.Frames,E.AlarmFrames,E.AvgScore,E.MaxScore,E.Archived,E.LearnState from Events as E inner join Monitors as M on M.Id = E.MonitorId where M.Id = '$monitor' and E.Id > ?"; + my $filter_sql = ''; + for ( my $i = 1; $i <= $filter_terms{trms}; $i++ ) + { + my $conjunction_name = "cnj$i"; + my $obracket_name = "obr$i"; + my $cbracket_name = "cbr$i"; + my $attr_name = "attr$i"; + my $op_name = "op$i"; + my $value_name = "val$i"; + if ( $filter_terms{$conjunction_name} ) + { + $filter_sql .= " ".$filter_terms{$conjunction_name}." "; + } + if ( $filter_terms{$obracket_name} ) + { + $filter_sql .= str_repeat( "(", $filter_terms{$obracket_name} ); + } + if ( $filter_terms{$attr_name} ) + { + if ( $filter_terms{$attr_name} eq 'DateTime' ) + { + my $dt_val = strtotime( $filter_terms{$value_name} ); + if ( !$dt_val ) + { + printf( STDERR "Error, unable to parse date '$filter_terms{$value_name}', skipping\n" ); + next FILTER; + } + $filter_sql .= "E.StartTime ".$filter_terms{$op_name}." from_unixtime( $dt_val )"; + } + elsif ( $filter_terms{$attr_name} eq 'Date' ) + { + my $dt_val = strtotime( $filter_terms{$value_name} ); + $filter_sql .= "to_days( E.StartTime ) ".$filter_terms{$op_name}." to_days( from_unixtime( $dt_val ) )"; + } + elsif ( $filter_terms{$attr_name} eq 'Time' ) + { + my $dt_val = strtotime( $filter_terms{$value_name} ); + $filter_sql .= "extract( hour_second from E.StartTime ) ".$filter_terms{$op_name}." extract( hour_second from from_unixtime( $dt_val ) )"; + } + elsif ( $filter_terms{$attr_name} eq 'Weekday' ) + { + my $dt_val = strtotime( $filter_terms{$value_name} ); + $filter_sql .= "weekday( E.StartTime ) ".$filter_terms{$op_name}." weekday( from_unixtime( $dt_val ) )"; + } + elsif ( $filter_terms{$attr_name} eq 'Archived' ) + { + $filter_sql .= "E.Archived = ".$filter_terms{$value_name}; + } + else + { + $filter_sql .= "E.".$filter_terms{$attr_name}." ".$filter_terms{$op_name}." ".$filter_terms{$value_name}; + } + } + if ( $filter_terms{$cbracket_name} ) + { + $filter_sql .= str_repeat( ")", $filter_terms{$cbracket_name} ); + } + } + if ( $filter_sql ) + { + $sql .= " and ( $filter_sql )"; + } + #print $sql."\n"; + $filter_data->{Sql} = $sql; + push( @filters, $filter_data ); + } + $sth->finish(); + + return( \@filters ); +}