From: Dirk Koopman Date: Sat, 12 Jul 2014 22:51:48 +0000 (+0100) Subject: Add logging modules and improve logger X-Git-Url: http://dxcluster.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=9a9679bb16f6676d568db58d46ac0bbe85da2a4c;p=dweather.git Add logging modules and improve logger Add some prototype support for LOOP2 records, but whether it is actually worth it is a cause of some speculation as stuff gets erased from memory too easily. --- diff --git a/.gitignore b/.gitignore index f020ecd..4336fee 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ MYMETA* Makefile logs pm_to_blib +\#* diff --git a/Debug.pm b/Debug.pm new file mode 100644 index 0000000..5bf2b5c --- /dev/null +++ b/Debug.pm @@ -0,0 +1,155 @@ +# +# The system variables - those indicated will need to be changed to suit your +# circumstances (and callsign) +# +# Copyright (c) 1998 - Dirk Koopman G1TLH +# +# This library is free software; you can redistribute it and/or +# modify it under the same terms as Perl itself. +# + +package Debug; + +require Exporter; + +@ISA = qw(Exporter); +@EXPORT = qw(dbginit dbg dbgadd dbgsub dbglist dbgdump isdbg dbgclose confess croak cluck carp); +$VERSION = 1.23; + +use strict; +use vars qw(%dbglevel $fp); + +use SMGLog (); +use Carp qw(cluck); + +%dbglevel = (); +$fp = undef; + +# Avoid generating "subroutine redefined" warnings with the following +# hack (from CGI::Carp): +if (!defined $DB::VERSION) { + local $^W=0; + eval qq( sub confess { + \$SIG{__DIE__} = 'DEFAULT'; + Debug::dbg(\$@, Carp::shortmess(\@_)); + exit(-1); + } + sub croak { + \$SIG{__DIE__} = 'DEFAULT'; + Debug::dbg(\$@, Carp::longmess(\@_)); + exit(-1); + } + sub carp { Debug::dbg(Carp::shortmess(\@_)); } + sub cluck { Debug::dbg(Carp::longmess(\@_)); } + ); + + CORE::die(Carp::shortmess($@)) if $@; +} else { + eval qq( sub confess { Carp::confess(\@_); } + sub cluck { Carp::cluck(\@_); } + sub carp { Carp::cluck(\@_); } + ); +} + +dbginit(); + +sub dbg +{ + my $t = time; + my $ts = sprintf("%02d:%02d:%02d", (gmtime($t))[2,1,0]); + for (@_) { + my $r = $_; + chomp $r; + my @l = split /\n/, $r; + for (@l) { + s/([\x00-\x08\x0B-\x1f\x7f-\xff])/uc sprintf("%%%02x",ord($1))/eg; +# print "$_\n" if defined \*STDOUT; + $fp->writeunix($t, "$ts $_"); + } + } +} + +sub dbginit +{ + # add sig{__DIE__} handling + if (!defined $DB::VERSION) { + $SIG{__WARN__} = sub { dbg($@, Carp::shortmess(@_)); }; + $SIG{__DIE__} = sub { dbg($@, Carp::longmess(@_)); }; + } + + $fp = SMGLog->new('debug', 'log', 'd'); +} + +sub dbgclose +{ + $SIG{__DIE__} = $SIG{__WARN__} = 'DEFAULT'; + $fp->close() if $fp; + undef $fp; +} + +sub dbgdump +{ + my $m = shift; + + foreach my $l (@_) { + my $p = $m; + for (my $o = 0; $o < length $l; $o += 16) { + my $c = substr $l, $o, 16; + my $h = unpack "H*", $c; + $c =~ s/[\x00-\x1f\x7f-\xff]/./g; + my $left = 16 - length $c; + $h .= ' ' x (2 * $left) if $left > 0; + dbg($p . sprintf("%4d:", $o) . "$h $c"); + $p = ' ' x (length $p); + } + } +} + +sub dbgadd +{ + my $entry; + + foreach $entry (@_) { + $dbglevel{$entry} = 1; + } +} + +sub dbgsub +{ + my $entry; + + foreach $entry (@_) { + delete $dbglevel{$entry}; + } +} + +sub dbglist +{ + return keys (%dbglevel); +} + +sub isdbg +{ + return undef unless $fp; + return $dbglevel{$_[0]}; +} + +sub shortmess +{ + return Carp::shortmess(@_); +} + +sub longmess +{ + return Carp::longmess(@_); +} + +1; +__END__ + + + + + + + diff --git a/SMGLog.pm b/SMGLog.pm new file mode 100644 index 0000000..ddff250 --- /dev/null +++ b/SMGLog.pm @@ -0,0 +1,205 @@ +# +# the general purpose logging machine +# +# This module is designed to allow you to log stuff in SMG format +# +# The idea is that you give it a prefix which is a directory and then +# the system will log stuff to a directory structure which looks like:- +# +# ./logs//yyyy/mmdd.[log|new("sys_log"); +} + +# create a log object that contains all the useful info needed +# prefix is the main directory off of the data directory +# suffix is the suffix after the month/day +sub new +{ + my ($pkg, $prefix, $suffix) = @_; + my $ref = {}; + my $dir = "$path/$prefix"; + $ref->{prefix} = $dir; + $ref->{suffix} = $suffix || 'log'; + + # make sure the directory exists + mkpath($dir, 0, 0777) unless -d $dir; + die "cannot create or access $dir $!" unless -d $dir; + + my $self = bless $ref, $pkg; + $open{$self} = $self; + return $self; +} + +sub mode +{ + my $self = shift; + $self->{mode} = shift if @_; + return $self->{mode}; +} + +# open the appropriate data file +sub open +{ + my ($self, $dayno, $mode) = @_; + + my ($year, $month, $day) = (gmtime($dayno * 86400))[5,4,3]; + $year += 1900; + $month += 1; + + # if we are writing, check that the directory exists + if (defined $mode) { + my $dir = "$self->{prefix}/$year"; + mkdir($dir, 0777) if ! -e $dir; + } + + $self->{fn} = sprintf "$self->{prefix}/$year/%02d%02d", $month, $day; + $self->{fn} .= ".$self->{suffix}" if $self->{suffix}; + + $self->{mode} = $mode || 'r'; + + my $fh = new IO::File $self->{fn}, $mode, 0666; + return unless $fh; + + $fh->autoflush(0) if $mode ne 'r'; # disable autoflushing if writable + $self->{fh} = $fh; + + $self->{year} = $year; + $self->{month} = $month; + $self->{day} = $day; + $self->{dayno} = $dayno; + +# DXDebug::dbg("dxlog", "opening $self->{fn}\n"); + + return $self->{fh}; +} + +# open the previous log file in sequence +sub openprev +{ + my $self = shift; + return $self->open($self->{dayno} - 1, @_); +} + +# open the next log file in sequence +sub opennext +{ + my $self = shift; + return $self->open($self->{dayno} + 1, @_); +} + +# write (actually append) to a file, opening new files as required +sub write +{ + my ($self, $dayno, $line) = @_; + if (!$self->{fh} || $self->{mode} ne "r" || $dayno != $self->{dayno}) { + $self->open($dayno, ">>") or confess "can't open $self->{fn} $!"; + } + + return $self->{fh}->print("$line\n"); +} + +# read a line from an opened file +sub read +{ + my $self = shift; + confess "can't read $self->{fh} $!" unless $self->{fh}; + return $self->{fh}->getline; +} + +# write (actually append) using the current date to a file, opening new files as required +sub writenow +{ + my ($self, $line) = @_; + my $dayno = int (time / 86400); + return $self->write($dayno, $line); +} + +# write (actually append) using a unix time to a file, opening new files as required +sub writeunix +{ + my ($self, $t, $line) = @_; + my $dayno = int ($t / 86400); + return $self->write($dayno, $line); +} + +# close the log file handle +sub close +{ + my $self = shift; + undef $self->{fh}; # close the filehandle + delete $self->{fh}; +} + +sub DESTROY +{ + my $self = shift; + + delete $open{$self}; + undef $self->{fh}; # close the filehandle + delete $self->{fh} if $self->{fh}; +} + +sub flush +{ + $_[0]->{fh}->flush if $_[0]->{fh}; +} + +sub flushall +{ + foreach my $v (values %open) { + $v->flush; + } +} + +sub flush_all { goto &flushall } + + +sub Log +{ + my $l = ref $_[0] ? shift : $log; + return unless $l; + my $t = time; + my $ts = sprintf("%02d:%02d:%02d", (gmtime($t))[2,1,0]); + $l->writeunix($t, "$ts $_") for @_; +} + +sub LogDbg +{ + Log(@_); + Debug::dbg(@_) if Debug::isdbg('chan'); +} + +1; diff --git a/Serial.pm b/Serial.pm new file mode 100644 index 0000000..26b9c2e --- /dev/null +++ b/Serial.pm @@ -0,0 +1,100 @@ +# +# Module to do serial handling on perl FileHandles +# + +package Serial; + +use POSIX qw(:termios_h); +use Fcntl; +use Scalar::Util qw(weaken); + + +@ISA = qw(IO::File); +$VERSION = 1.3; + +use strict; + +# Linux-specific Baud-Rates +use constant B57600 => 0010001; +use constant B115200 => 0010002; +use constant B230400 => 0010003; +use constant B460800 => 0010004; +use constant CRTSCTS => 020000000000; + +sub new +{ + my $pkg = shift; + my $class = ref $pkg || $pkg; + my $device = shift || "/dev/ttyS0"; + + my $self = $pkg->SUPER::new($device, O_RDWR|O_NOCTTY|O_EXCL|O_NDELAY) || return; + + # get my attributes + $$self->{ORIGTERM} = POSIX::Termios->new(); + my $term = $$self->{TERM} = POSIX::Termios->new(); + $$self->{ORIGTERM}->getattr(fileno($self)); + $term->getattr(fileno($self)); + my ($speed) = grep {/^\d+$/} @_; + my $baud; + { + no strict 'refs'; + $baud = &{'POSIX::B' . $speed}; + } + $term->setispeed($baud); + $term->setospeed($baud); + + my $cflag = $term->getcflag(); my $lflag = $term->getlflag(); + my $oflag = $term->getoflag(); my $iflag = $term->getiflag(); + + # set raw + ######################################################################## + $iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP|INLCR|IGNCR|ICRNL|IXON); + $oflag &= ~OPOST; + $lflag &= ~(ECHO|ECHONL|ICANON|ISIG); + $cflag &= ~(CSIZE|PARENB|HUPCL); + ######################################################################### + # + + $cflag |= CLOCAL|CREAD; + $cflag |= (grep {/^cs7$/i} @_) ? CS7 : CS8; + if (my ($parity) = grep {/^(odd|even)$/i} $@) { + $cflag |= PARENB; + $cflag |= PARODD if $parity =~ /odd/i; + } + $cflag |= CRTSCTS if grep /rtscts$/, $@; + $term->setcflag($cflag); $term->setlflag($lflag); + $term->setoflag($oflag); $term->setiflag($iflag); + $term->setattr(fileno($self), TCSANOW); + return $self; +} + +sub getattr +{ + my $self = shift; + $$self->{TERM}->getattr; + return $$self->{TERM}; +} + +sub setattr +{ + my $self = shift; + my $attr = shift || $$self->{TERM}; + $attr->setattr(fileno($self), &POSIX::TCSANOW); +} + +sub close +{ + my $self = shift; + $self->setattr(delete $$self->{ORIGTERM}); + $self->SUPER::close; +} + +sub DESTROY +{ + my $self = shift; + if (exists $$self->{ORIGTERM}) { + $self->close; + } +} + +1; diff --git a/loop.pl b/loop.pl index 9420e2b..8a33ba7 100755 --- a/loop.pl +++ b/loop.pl @@ -3,11 +3,12 @@ use strict; use v5.10.1; -use DBI; use Serial; use Mojo::IOLoop; use Mojo::IOLoop::Stream; use Mojo::JSON qw(decode_json encode_json); +use Debug; +use SMGLog; my $devname = "/dev/davis"; my $rain_mult = 0.1; # 0.1 or 0.2 mm or 0.01 inches @@ -66,12 +67,15 @@ $bar_trend{60} = "Rising Rapidly"; #$SIG{TERM} = $SIG{INT} = sub {Mojo::IOLoop->stop if Mojo::IOLoop->is_running && !$DB::VERSION}; +my $dlog = SMGLog->new("day"); +$dlog->mode('a'); + my $s = do_open($devname); start_loop(); -Mojo::IOLoop->start unless Mojo::IOLoop->is_running; +Mojo::IOLoop->recurring(0.1 => sub { $dlog->flushall }); -$s->close; +Mojo::IOLoop->start unless Mojo::IOLoop->is_running; exit 0; @@ -82,9 +86,9 @@ sub on_read $d =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; # say "read added '$d' buf lth=" . length $buf if $dbg; if ($state eq 'waitnl' && $buf =~ /[\cJ\cM]+/) { - undef $tid; + Mojo::IOLoop->remove($tid); undef $buf; - $s->write("LOOP 1\n"); + $s->write("LPS 1 1\n"); chgstate("waitloop"); } elsif ($state eq "waitloop") { if ($buf =~ /\x06/) { @@ -107,7 +111,7 @@ sub start_loop say "writing \\n" if $dbg; $s->write("\n"); - $tid = Mojo::IOLoop->timer(0.6 => sub {say "writing \\n" if $dbg; $s->write("\n")}); + $tid = Mojo::IOLoop->recurring(0.6 => sub {say "writing \\n" if $dbg; $s->write("\n")}); chgstate("waitnl"); } @@ -133,11 +137,15 @@ sub do_open $str->on(read=>sub {on_read(@_)}); $str->start; - $rid = Mojo::IOLoop->recurring(2.5 => sub {start_loop() if !$state || $state eq "waitnl";}); + $rid = Mojo::IOLoop->recurring(2.5 => sub { + start_loop() if !$state || $state eq "waitnl"; + }); return $str; } +my $last_time; + sub process { my $blk = shift; @@ -149,60 +157,72 @@ sub process my $t; my %h; - - #$h{'next_rec'} = unpack("s", substr $blk,5,2); - - $h{'Barometric_Trend'} = unpack("C", substr $blk,3,1); - $h{'Barometric_Trend_txt'} = $bar_trend{$h{'Barometric_Trend'}}; + # Common ones + $h{Pressure_Trend} = unpack("C", substr $blk,3,1); + $h{Pressure_Trend_txt} = $bar_trend{$h{'Pressure_Trend'}}; $t = unpack("s", substr $blk,7,2) / 1000; - $h{'Barometric_Press_mb'} = sprintf("%.0f",$t*33.8637526); - + $h{Pressure} = sprintf("%.0f",in2mb($t))+0; $t = unpack("s", substr $blk,9,2) / 10; - $h{'Air_Temp_Inside_c'} = sprintf("%.1f",($t - 32) * 5/9); - my $tf = unpack("s", substr $blk,12,2) / 10; - $h{'Air_Temp_Outside_c'} = sprintf("%.1f",($tf - 32) * 5/9); - - $h{'Wind_Speed_mph'} = unpack("C", substr $blk,14,1); - $h{'Wind_Speed_mps'} = sprintf("%.1f",$h{'Wind_Speed_mph'}*0.44704); - $h{'Wind_Speed_10min_Avg_mph'} = unpack("C", substr $blk,15,1); - $h{'Wind_Speed_10min_Avg_mps'} = sprintf("%.1f",$h{'Wind_Speed_10min_Avg_mph'}*0.44704); - $h{'Wind_Dir'} = unpack("s", substr $blk,16,2); - + $h{Temp_In} = sprintf("%.1f", f2c($t))+0; + + $t = unpack("s", substr $blk,12,2) / 10; + $h{Temp_Out} = sprintf("%.1f", f2c($t))+0; - $h{'Humidity_Outside'} = unpack("C", substr $blk,33,1); - $h{'Humidity_Inside'} = unpack("C", substr $blk,11,1); - $h{'Dew_Point'} = dew_point($h{Air_Temp_Outside_c}, $h{'Humidity_Outside'}); + $t = unpack("C", substr $blk,14,1); + $h{Wind} = sprintf("%.1f",mph2mps($t))+0; + $h{Dir} = unpack("s", substr $blk,16,2)+0; - $h{'UV'} = unpack("C", substr $blk,43,1); - $h{'Solar'} = unpack("s", substr $blk,44,2); # watt/m**2 - $h{'Rain_Rate'} = unpack("s", substr $blk,41,2) * $rain_mult; - $h{'Rain_Day'} = unpack("s", substr $blk,50,2) * $rain_mult; - $h{'Rain_Month'} = unpack("s", substr $blk,52,2) * $rain_mult; - $h{'Rain_Year'} = unpack("s", substr $blk,54,2) * $rain_mult; + $h{'Humidity_Out'} = unpack("C", substr $blk,33,1)+0; + $h{'Humidity_In'} = unpack("C", substr $blk,11,1)+0; - $h{'ET_Day'} = unpack("s", substr $blk,56,2)/1000; - $h{'ET_Month'} = unpack("s", substr $blk,58,2)/100; - $h{'ET_Year'} = unpack("s", substr $blk,60,2)/100; + $t = unpack("C", substr $blk,43,1)+0; + $h{'UV'} = $t unless $t >= 255; + $t = unpack("s", substr $blk,44,2)+0; # watt/m**2 + $h{'Solar'} = $t unless $t >= 32767; - #$h{'Alarms_Inside'} = unpack("b8", substr $blk,70,1); - #$h{'Alarms_Rain'} = unpack("b8", substr $blk,70,1); - #$h{'Alarms_Outside'} = unpack("b8", substr $blk,70,1); + $h{'Rain_Rate'} = sprintf("%0.1f",unpack("s", substr $blk,41,2) * $rain_mult)+0; + $h{'Rain_Day'} = sprintf("%0.1f", unpack("s", substr $blk,50,2) * $rain_mult)+0; - $h{'Batt_TX_OK'} = (unpack("C", substr $blk,86,1)+0) ^ 1; - $h{'Batt_Console'} = unpack("s", substr $blk,87,2) * 0.005859375; + # what sort of packet is it? - $h{'Forecast_Icon'} = unpack("C", substr $blk,89,1); - $h{'Forecast_Rule'} = unpack("C", substr $blk,90,1); + $t = unpack("C", substr $blk,4,1); + if ($t) { - $h{'Sunrise'} = sprintf( "%04d", unpack("S", substr $blk,91,2) ); - $h{'Sunrise'} =~ s/(\d{2})(\d{2})/$1:$2/; - $h{'Sunset'} = sprintf( "%04d", unpack("S", substr $blk,93,2) ); - $h{'Sunset'} =~ s/(\d{2})(\d{2})/$1:$2/; + # Newer LOOP2 packet + $t = unpack("C", substr $blk,18,2); + $h{Wind_Avg_10} = sprintf("%.1f",mph2mps($t/10))+0; + $t = unpack("C", substr $blk,20,2); + $h{Wind_Avg_2} = sprintf("%.1f",mph2mps($t/10))+0; + $t = unpack("C", substr $blk,22,2); + $h{Wind_Gust_10} = sprintf("%.1f",mph2mps($t/10))+0; + + $h{Dir_Avg_10} = unpack("C", substr $blk,24,2)+0; + $t = unpack("C", substr $blk,30,2); + $h{Dew_Point} = sprintf("%0.1f", f2c($t))+0; - #my $nl = ord substr $blk,95,1; - #my $cr = ord substr $blk,96,1; + } else { + + # Older LOOP packet + $t = unpack("C", substr $blk,15,1); + $h{Wind_Avg_10} = sprintf("%.1f",mph2mps($t))+0; + $h{'Dew_Point'} = sprintf("%0.1f", dew_point($h{Temp_Out}, $h{'Humidity_Out'}))+0; + + $h{'Rain_Month'} = sprintf("%0.1f", unpack("s", substr $blk,52,2) * $rain_mult)+0; + $h{'Rain_Year'} = sprintf("%0.1f", unpack("s", substr $blk,54,2) * $rain_mult)+0; + + + $h{'Batt_TX_OK'} = (unpack("C", substr $blk,86,1)+0) ^ 1; + $h{'Batt_Console'} = sprintf("%0.2f", unpack("s", substr $blk,87,2) * 0.005859375)+0; + $h{'Forecast_Icon'} = unpack("C", substr $blk,89,1); + $h{'Forecast_Rule'} = unpack("C", substr $blk,90,1); + + $h{'Sunrise'} = sprintf( "%04d", unpack("S", substr $blk,91,2) ); + $h{'Sunrise'} =~ s/(\d{2})(\d{2})/$1:$2/; + $h{'Sunset'} = sprintf( "%04d", unpack("S", substr $blk,93,2) ); + $h{'Sunset'} =~ s/(\d{2})(\d{2})/$1:$2/; + } my $crc = unpack "%n", substr($blk,97,2); my $crc_calc = CRC_CCITT($blk); @@ -210,21 +230,25 @@ sub process if ($crc_calc==0) { my $o = gen_hash_diff($last_reading, \%h); $last_reading = \%h; - if (time % 60 == 0) { - my $oo = {t => time, r =>\%h}; - say encode_json($oo); + my $t = time; + my $j; + my $s; + if ($t >= $last_time + 60) { + $j = encode_json(\%h); + $s = qq|{"t":$t,"m":$j}|; + $last_time = $t; } elsif ($o) { - my $oo = {t => time, r =>$o}; - say encode_json($oo); + $j = encode_json($o); + $s = qq|{"t":$t,"r":$j}|; + } + if ($s) { + say $s; + $dlog->writenow($s); } } else { say "CRC check failed for LOOP data!"; return 1; } - #delete @h{'crc', 'crc_calc', 'next_rec'}; - #delete($h{crc})||die"cant delete crc"; - #delete($h{crc_calc})||die"cant delete crc_calc"; - #delete($h{next_rec})||die"cant delete next_rec"; } sub gen_hash_diff @@ -252,9 +276,16 @@ sub dew_point # Accurate to 1 degree C for humidities > 50 % # http://en.wikipedia.org/wiki/Dew_point - my $dew_point = $temp - ( (100 - $rh)/5 ); + my $dewpoint = $temp - ((100 - $rh) / 5); + + # this is the more complete one (which doesn't work) + #my $a = 6.1121; + #my $b = 18.678; + #my $c = 257.14; + #my $ytrh = log(($rh/100) + ($b * $temp) / ($c + $temp)); + #my $dewpoint = ($c * $ytrh) / ($b - $ytrh); - return $dew_point; + return $dewpoint; } sub CRC_CCITT @@ -280,3 +311,17 @@ sub CRC_CCITT return $crc; } +sub f2c +{ + return ($_[0] - 32) * 5/9; +} + +sub mph2mps +{ + return $_[0] * 0.44704; +} + +sub in2mb +{ + return $_[0] * 33.8637526; +}