#!/usr/bin/perl
# $Id$
# logwatch script to process weewx log files
# Copyright 2013 Matthew Wall
#
# Revision History
#  0.4 12oct13
#   * recognize more fousb log output
#   * recognize more ws28xx log output
#   * track forecasting counts
#  0.3 09oct13
#   * match cheetahgenerator
#   * match failed restful uploads
#   * recognize forecast events
#   * match weewx HUPs
#   * recognize new driver startup diagnostics
#   * recognize new weewx wxengine startup diagnostics
#   * recognize ws28xx driver entries
#  0.2 03jan13
#   * better labels for counts
#  0.1 01jan13
#   * initial release

use strict;

my $STARTUPS = 'wxengine: startups';
my $HUP_RESTARTS = 'wxengine: restart from HUP';
my $ARCHIVE_RECORDS_ADDED = 'archive: records added';
my $IMAGES_GENERATED = 'genimages: images generated';
my $FILES_GENERATED = 'filegenerator: files generated';
my $FILES_COPIED = 'reportengine: files copied';
my $RECORDS_PUBLISHED = 'restful: records published';
my $RECORDS_SKIPPED = 'restful: records skipped';
my $FOUSB_UNSTABLE_READS = 'fousb: unstable reads';
my $FOUSB_LOST_LOG_SYNC = 'fousb: lost log sync';
my $FOUSB_LOST_SYNC = 'fousb: lost sync';
my $FOUSB_MISSED_DATA = 'fousb: missed data';
my $FOUSB_STATION_SYNC = 'fousb: station sync';
my $FORECAST_RECORDS = 'forecast: records generated';
my $FORECAST_PRUNINGS = 'forecast: prunings';
my $FORECAST_DOWNLOADS = 'forecast: downloads';
my %counts = (
    $STARTUPS, 0,
    $HUP_RESTARTS, 0,
    $ARCHIVE_RECORDS_ADDED, 0,
    $IMAGES_GENERATED, 0,
    $FILES_GENERATED, 0,
    $FILES_COPIED, 0,
    $RECORDS_PUBLISHED, 0,
    $RECORDS_SKIPPED, 0,
    $FOUSB_UNSTABLE_READS, 0,
    $FOUSB_LOST_LOG_SYNC, 0,
    $FOUSB_LOST_SYNC, 0,
    $FOUSB_MISSED_DATA, 0,
    $FOUSB_STATION_SYNC, 0,
    $FORECAST_RECORDS, 0,
    $FORECAST_PRUNINGS, 0,
    $FORECAST_DOWNLOADS, 0,
);
my $RECORDS_FAILED = 'restful: publish failed';
my %errors;
my @unmatched = ();

while(defined($_ = <STDIN>)) {
    chomp;
    if (/Archive: added archive record/) {
        $counts{$ARCHIVE_RECORDS_ADDED} += 1;
    } elsif (/genimages: Generated (\d+) images/) {
        $counts{$IMAGES_GENERATED} += $1;
    } elsif (/filegenerator: generated (\d+)/ ||
             /cheetahgenerator: generated (\d+)/) {
        $counts{$FILES_GENERATED} += $1;
    } elsif (/wxengine: Starting up weewx version/) {
        $counts{$STARTUPS} += 1;
    } elsif (/wxengine: Received signal HUP/) {
        $counts{$HUP_RESTARTS} += 1;
    } elsif (/reportengine: copied (\d+) files/) {
        $counts{$FILES_COPIED} += $1;
    } elsif (/fousb: unstable read: blocks differ/) {
        $counts{$FOUSB_UNSTABLE_READS} += 1;
    } elsif (/fousb: lost log sync/) {
        $counts{$FOUSB_LOST_LOG_SYNC} += 1;
    } elsif (/fousb: lost sync/) {
        $counts{$FOUSB_LOST_SYNC} += 1;
    } elsif (/fousb: missed data/) {
        $counts{$FOUSB_MISSED_DATA} += 1;
    } elsif (/fousb: synchronising to the weather station/) {
        $counts{$FOUSB_STATION_SYNC} += 1;
    } elsif (/forecast: .* generated 1 forecast record/) {
        $counts{$FORECAST_RECORDS} += 1;
    } elsif (/forecast: .* got (\d+) forecast records/) {
        $counts{$FORECAST_RECORDS} += $1;
    } elsif (/forecast: .* deleted forecasts/) {
        $counts{$FORECAST_PRUNINGS} += 1;
    } elsif (/forecast: .* downloading forecast/) {
        $counts{$FORECAST_DOWNLOADS} += 1;
    } elsif (/restful: Skipped record/) {
        $counts{$RECORDS_SKIPPED} += 1;
    } elsif (/restful: Published record/) {
        $counts{$RECORDS_PUBLISHED} += 1;
    } elsif (/genimages: aggregate interval required for aggregate type/ ||
             /genimages: line type \S+ skipped/) {
        $errors{$_} = $errors{$_} ? $errors{$_} + 1 : 1;
    } elsif (/restful: Unable to publish record/) {
        my $key = $RECORDS_FAILED;
        if (/restful: Unable to publish record \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \S\S\S \(\d+\) to (\S+)/) {
            $key .= ' to site ' . $1;
        }
        $errors{$key} = $errors{$key} ? $errors{$key} + 1 : 1;
    } elsif (/reportengine: Running reports for latest time/ ||
             /reportengine: Found configuration file/ ||
             /reportengine: FTP upload not requested/ ||
             /reportengine: Running report / ||  # only when debug=1
             /restful: station will register with/ ||
             /restful: Registration interval/ ||
             /\*\*\*\*  Registration interval/ ||
             /restful: Registration successful/ ||
             /restful: Attempting to register/ ||
             /stats: Back calculated schema/ ||
             /stats: Backfilling stats database/ ||
             /stats: backfilled \d+ days of statistics/ ||
             /stats: stats database up to date/ ||
             /stats: Created schema for statistical database/ ||
             /stats: Schema exists with/ ||
             /\*\*\*\*  \'station\'/ ||
             /\*\*\*\*  Waiting 60 seconds then retrying/ ||
             /wxengine: Station does not support reading the time/ ||
             /wxengine: Starting main packet loop/ ||
             /wxengine: Keyboard interrupt/ ||
             /wxengine: Shut down StdReport thread/ ||
             /wxengine: Shut down StdRESTful thread/ ||
             /wxengine: Loading service/ ||
             /wxengine: Finished loading service/ ||
             /wxengine: Using archive interval of/ ||
             /wxengine: Using archive database/ ||
             /wxengine: Using configuration file/ ||
             /wxengine: Using stats database/ ||
             /wxengine: Using station hardware archive interval/ ||
             /wxengine: Using config file archive interval of/ ||
             /wxengine: Record generation will be attempted in/ ||
             /wxengine: StdConvert target unit is/ ||
             /wxengine: Data will not be posted to/ ||
             /wxengine: Data will be posted to / ||
             /wxengine: Started thread for RESTful upload sites./ ||
             /wxengine: No RESTful upload sites/ ||
             /wxengine: retrying/ ||
             /wxengine: Loading station type/ ||
             /wxengine: Initializing weewx version/ ||
             /wxengine: Using Python/ ||
             /wxengine: Terminating weewx version/ ||
             /wxengine: pid file is / ||
             /wxengine: Use LOOP data in/ ||
             /wxengine: Received signal/ ||
             /cheetahgenerator: skip/ ||
             /fousb: found station on USB/ ||
             /fousb: altitude is/ ||
             /fousb: archive interval is/ ||
             /fousb: pressure offset is/ ||
             /fousb: polling mode is/ ||
             /fousb: polling interval is/ ||
             /fousb: using \S+ polling mode/ ||
             /fousb: ptr changed/ ||
             /fousb: new ptr/ ||
             /fousb: new data/ ||
             /fousb: live synchronised/ ||
             /fousb: log synchronised/ ||
             /fousb: log extended/ ||
             /fousb: delay/ ||
             /fousb: avoid/ ||
             /fousb: setting sensor clock/ ||
             /fousb: setting station clock/ ||
             /fousb: returning archive record/ ||
             /fousb: packet timestamp/ ||
             /fousb: log timestamp/ ||
             /fousb: found \d+ archive records/ ||
             /fousb: get \d+ records since/ ||
             /fousb: synchronised to/ ||
             /fousb: pressures:/ ||
             /ws28xx: MainThread: driver version is/ ||
             /ws28xx: MainThread: frequency is/ ||
             /ws28xx: MainThread: altitude is/ ||
             /ws28xx: MainThread: pressure offset is/ ||
             /ws28xx: MainThread: found transceiver on USB/ ||
             /ws28xx: MainThread: manufacturer: LA CROSSE TECHNOLOGY/ ||
             /ws28xx: MainThread: product: Weather Direct Light Wireless/ ||
             /ws28xx: MainThread: interface/ ||
             /ws28xx: MainThread: base frequency/ ||
             /ws28xx: MainThread: frequency correction/ ||
             /ws28xx: MainThread: adjusted frequency/ ||
             /ws28xx: MainThread: transceiver identifier/ ||
             /ws28xx: MainThread: transceiver serial/ ||
             /ws28xx: MainThread: execute/ ||
             /ws28xx: MainThread: setState/ ||
             /ws28xx: MainThread: setPreamPattern/ ||
             /ws28xx: MainThread: setRX/ ||
             /ws28xx: MainThread: readCfgFlash/ ||
             /ws28xx: MainThread: setFrequency/ ||
             /ws28xx: MainThread: setDeviceID/ ||
             /ws28xx: MainThread: setTransceiverSerialNumber/ ||
             /ws28xx: MainThread: setCommModeInterval/ ||
             /ws28xx: MainThread: frequency registers/ ||
             /ws28xx: MainThread: initTransceiver/ ||
             /ws28xx: MainThread: startRFThread/ ||
             /ws28xx: MainThread: stopRFThread/ ||
             /ws28xx: MainThread: detach kernel driver/ ||
             /ws28xx: MainThread: release USB interface/ ||
             /ws28xx: MainThread: claiming USB interface/ ||
             /ws28xx: MainThread: CCommunicationService.init/ ||
             /ws28xx: RFComm: console is paired to device/ ||
             /ws28xx: RFComm: starting rf communication/ ||
             /ws28xx: RFComm: stopping rf communication/ ||
             /ws28xx: RFComm: setTX/ ||
             /ws28xx: RFComm: setRX/ ||
             /ws28xx: RFComm: setState/ ||
             /ws28xx: RFComm: getState/ ||
             /ws28xx: RFComm: setFrame/ ||
             /ws28xx: RFComm: getFrame/ ||
             /ws28xx: RFComm: InBuf/ ||
             /ws28xx: RFComm: OutBuf/ ||
             /ws28xx: RFComm: generateResponse: sleep/ ||
             /ws28xx: RFComm: generateResponse: id/ ||
             /ws28xx: RFComm: handleCurrentData/ ||
             /ws28xx: RFComm: handleHistoryData/ ||
             /ws28xx: RFComm: handleNextAction/ ||
             /ws28xx: RFComm: handleConfig/ ||
             /ws28xx: RFComm: buildACKFrame/ ||
             /ws28xx: RFComm: buildTimeFrame/ ||
             /ws28xx: RFComm: buildConfigFrame/ ||
             /ws28xx: RFComm: setCurrentWeather/ ||
             /ws28xx: RFComm: setHistoryData/ ||
             /ws28xx: RFComm: setDeviceCS/ ||
             /ws28xx: RFComm: setRequestType/ ||
             /ws28xx: RFComm: setResetMinMaxFlags/ ||
             /ws28xx: RFComm: setLastStatCache/ ||
             /ws28xx: RFComm: setLastConfigTime/ ||
             /ws28xx: RFComm: setLastHistoryIndex/ ||
             /ws28xx: RFComm: setLastHistoryDataTime/ ||
             /ws28xx: RFComm: CCurrentWeatherData.read/ ||
             /ws28xx: RFComm: CWeatherStationConfig.read/ ||
             /ws28xx: RFComm: CHistoryDataSet.read/ ||
             /ws28xx: RFComm: testConfigChanged/ ||
             /ws28xx: RFComm: SetTime/ ||
             /forecast: .* starting thread/ ||
             /forecast: .* terminating thread/ ||
             /forecast: .* not yet time to do the forecast/ ||
             /forecast: .* last forecast issued/ ||
             /forecast: .* using table/ ||
             /forecast: .* tstr=/ ||
             /forecast: .* interval=\d+ max_age=/ ||
             /forecast: NWSThread: NWS: forecast matrix/ ||
             /forecast: XTideThread: XTide: tide matrix/ ||
             /forecast: XTideThread: XTide: generating tides/ ||
             /emoncms: Failed upload attempt/ ||
             /emoncms: Failed upload to EmonCMS/ ||
             /\*\*\*\*  Failed upload to EmonCMS/ ||
             /seg: Failed upload attempt/ ||
             /seg: Failed upload to SmartEnergyGroups/ ||
             /\*\*\*\*  Failed upload to SmartEnergyGroups/ ||
             /cosm: Failed upload attempt/ ||
             /cosm: Failed upload to COSM/ ||
             /\*\*\*\*  Failed upload to COSM/) {
        # ignore
    } elsif (! /weewx/) {
        # ignore
    } else {
        push @unmatched, $_;
    }
}

print "processing counts:\n";
foreach my $k (sort keys %counts) {
    next if $counts{$k} == 0;
    printf("  %-40s %6d\n", $k, $counts{$k});
}
print "\nerrors:\n";
foreach my $k (keys %errors) {
    printf("  %3d   %s\n", $errors{$k}, $k);
}

report("unmatched lines", \@unmatched) if $#unmatched >= 0;

exit 0;

sub report {
    my($label, $aref, $href) = @_;
    print "\n$label:\n";
    foreach my $x (@$aref) {
        my $str = $x;
        if ($href && $href->{$x} > 1) {
            $str .= " ($href->{$x} times)";
        }
        print "  $str\n";
    }
}
