]> dxcluster.org Git - dweather.git/blob - loop.pl
fix windrose and other tarting
[dweather.git] / loop.pl
1 #!/usr/bin/perl
2 use strict;
3
4 use v5.10.1;
5
6 use Mojolicious::Lite;
7 use Serial;
8 use Mojo::IOLoop;
9 use Mojo::IOLoop::Stream;
10 use Mojo::Transaction::WebSocket;
11 #use Mojo::JSON  qw(decode_json encode_json);
12 use JSON;
13 use Debug;
14 use SMGLog;
15 use Math::Round qw(nearest);
16 use File::Copy;
17 use Data::Random qw(rand_chars);
18 use IO::File;
19
20 use constant pi => 3.14159265358979; 
21
22 my $devname = "/dev/davis";
23 my $datafn = ".loop_data";
24 my $dataf;
25
26 my $poll_interval = 2.5;
27 my $rain_mult = 0.2;                            # 0.1 or 0.2 mm or 0.01 inches
28 my $tid;
29 my $rid;
30 our $did;
31 my $nlcount;
32 my $state = "ready";
33 my $buf;
34 my $dbg;
35 my $ser;                                                        # the serial port Mojo::IOLoop::Stream
36 my $last_min_h;
37 my $last_hour_h;
38
39 our $json = JSON->new->canonical(1);
40 our $WS = {};                                   # websocket connections
41
42 our $ld = {};
43 our @last10minsr = ();
44 our @last5daysh = ();
45 our $windmins = 2;                              # no of minutes of wind data for the windrose
46 our $histdays = 5;                              # no of days of (half)hour data to search for main graph
47 our $updatepermin = 60 / 2.5;   # no of updates per minute
48
49 our $loop_count;                                # how many LOOPs we have done, used as start indicator
50
51 our     @crc_table = (
52                                   0x0, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
53                                   0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
54                                   0x1231, 0x210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
55                                   0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
56                                   0x2462, 0x3443, 0x420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
57                                   0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
58                                   0x3653, 0x2672, 0x1611, 0x630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
59                                   0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
60                                   0x48c4, 0x58e5, 0x6886, 0x78a7, 0x840, 0x1861, 0x2802, 0x3823,
61                                   0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
62                                   0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0xa50, 0x3a33, 0x2a12,
63                                   0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
64                                   0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0xc60, 0x1c41,
65                                   0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
66                                   0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0xe70,
67                                   0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
68                                   0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
69                                   0x1080, 0xa1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
70                                   0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
71                                   0x2b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
72                                   0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
73                                   0x34e2, 0x24c3, 0x14a0, 0x481, 0x7466, 0x6447, 0x5424, 0x4405,
74                                   0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
75                                   0x26d3, 0x36f2, 0x691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
76                                   0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
77                                   0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x8e1, 0x3882, 0x28a3,
78                                   0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
79                                   0x4a75, 0x5a54, 0x6a37, 0x7a16, 0xaf1, 0x1ad0, 0x2ab3, 0x3a92,
80                                   0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
81                                   0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0xcc1,
82                                   0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
83                                   0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0xed1, 0x1ef0
84                                  );
85
86 our %bar_trend;
87
88 $bar_trend{-60} = "Falling Rapidly";
89 $bar_trend{196} = "Falling Rapidly";
90 $bar_trend{-20} = "Falling Slowly";
91 $bar_trend{236} = "Falling Slowly";
92 $bar_trend{0} = "Steady";
93 $bar_trend{20} = "Rising Slowly";
94 $bar_trend{60} = "Rising Rapidly";
95
96 our $ending = 0;
97
98 $SIG{TERM} = $SIG{INT} = sub {++$ending; Mojo::IOLoop->stop;};
99 $SIG{HUP} = 'IGNORE';
100
101
102 # WebSocket weather service
103 websocket '/weather' => sub {
104   my $c = shift;
105   my $msg = shift;
106   my $tx = $c->tx;
107
108   # Opened
109   app->log->debug('WebSocket opened.');
110   dbg 'WebSocket opened' if isdbg 'chan';
111   $WS->{$tx} = $tx; 
112
113   # send historical data
114   $c->send($ld->{lasthour_h}) if exists $ld->{lasthour_h};
115   $c->send($ld->{lastmin_h}) if exists $ld->{lastmin_h};
116
117   # disable timeout
118   $c->inactivity_timeout(3615);
119  
120   # Incoming message
121   $c->on(
122                  message => sub {
123                          my ($c, $msg) = @_;
124                          dbg "websocket: text $msg" if isdbg 'chan';
125                  },
126                  json => sub {
127                          my ($c, $msg) = @_;
128                          dbg "websocket: json $msg" if isdbg 'chan';
129                  }
130   );
131  
132   # Closed
133   $c->on(finish => sub {
134     my ($c, $code, $reason) = @_;
135     app->log->debug("WebSocket closed with status $code.");
136         dbg "websocket closed with status $code" if isdbg 'chan';
137         delete $WS->{$tx};
138   });
139 };
140
141 get '/' => {template => 'index'};
142
143
144 dbginit();
145 if (@ARGV) {
146         dbgadd(@ARGV);
147
148 dbgadd('chan');
149
150 dbg '***';
151 dbg "*** starting $0";
152 dbg '***';
153
154 read_ld();
155
156 my $tnow = time;
157 my $dayno = int ($tnow/86400);
158 for (my $i = 0-$histdays; $i < 0; ++$i ) {
159         push @last5daysh, grab_history(SMGLog->new("day"), "h", $tnow-(86400*$histdays), $dayno+$i+1); 
160 }
161 @last10minsr = map {my ($t, $js) = split(/\s/, $_, 2); $js} grab_history(SMGLog->new("debug"), "r", $tnow-(60*$windmins), $dayno);
162 dbg sprintf("last5days = %d last10mins = %d", scalar @last5daysh, scalar @last10minsr);
163
164 our $dlog = SMGLog->new("day");
165 dbg "before next tick";
166 Mojo::IOLoop->next_tick(sub { loop() });        
167 dbg "before app start";
168 app->secrets([qw(Here's something that's really seakrett)]);
169 app->start;
170 dbg "after app start";
171
172 write_ld();
173 $dataf->close if $dataf;
174 undef $dataf;
175
176
177 # move all the files along one 
178 cycle_loop_data_files();
179
180 dbg '***';
181 dbg "*** ending $0";
182 dbg '***';
183
184 exit 0;
185
186 ##################################################################################
187         
188 sub loop
189 {
190         dbg "last_min: " . scalar gmtime($ld->{last_min});
191         dbg "last_hour: " . scalar gmtime($ld->{last_hour});
192         
193         $did = Mojo::IOLoop->recurring(1 => sub {$dlog->flushall});
194         
195         do_reopen($devname);
196 }
197
198
199
200 sub on_read
201 {
202         my ($ser, $d) = @_;
203         $buf .= $d;
204         $d =~ s/([\%\x00-\x1f\x7f-\xff])/sprintf("%%%02X", ord($1))/eg; 
205         dbg "read added '$d' buf lth=" . length $buf if isdbg 'raw';
206         if ($state eq 'waitnl' && $buf =~ /[\cJ\cM]+/) {
207                 dbg "Got \\n" if isdbg 'state';
208                 Mojo::IOLoop->remove($tid) if $tid;
209                 undef $tid;
210                 undef $buf;
211                 $nlcount = 0;
212                 $ser->write("LPS 1 1\n");
213                 chgstate("waitloop");
214         } elsif ($state eq "waitloop") {
215                 if ($buf =~ /\x06/) {
216                         dbg "Got ACK 0x06" if isdbg 'state';
217                         chgstate('waitlooprec');
218                         undef $buf;
219                 }
220         } elsif ($state eq 'waitlooprec') {
221                 if (length $buf >= 99) {
222                         dbg "got loop record" if isdbg 'chan';
223                         chgstate('');
224                         process($buf);
225                         undef $buf;
226                 }
227         }
228 }
229
230 sub start_loop
231 {
232         $nlcount = 1;
233         dbg "start_loop writing $nlcount \\n" if isdbg 'state';
234         $ser->write("\n");
235         Mojo::IOLoop->remove($tid) if $tid;
236         undef $tid;
237         $tid = Mojo::IOLoop->recurring(0.6 => sub {
238                                                                            if (++$nlcount > 10) {
239                                                                                    dbg "\\n count > 10, closing connection" if isdbg 'chan';
240                                                                                    do_reopen($devname);
241                                                                                    return;
242                                                                            }
243                                                                            dbg "writing $nlcount \\n" if isdbg 'state'; 
244                                                                            $ser->write("\n");
245                                                                    });
246         chgstate("waitnl");
247 }
248
249 sub chgstate
250 {
251         dbg "state '$state' -> '$_[0]'" if isdbg 'state';
252         $state = $_[0];
253 }
254
255 sub do_reopen
256 {
257         my $name = shift;
258         dbg "do reopen on '$name' ending $ending";
259         unless ($ending) {
260                 $ser = do_open($name);
261                 start_loop();
262                 chgstate('');
263                 $nlcount = 0;
264                 Mojo::IOLoop->start unless Mojo::IOLoop->is_running;
265         }
266 }
267
268 sub do_open
269 {
270         my $name = shift;
271
272         $ser->close if $ser;
273         undef $ser;
274
275         my $ob = Serial->new($name, 19200) || die "$name $!\n";
276         dbg "streaming $name fileno(" . fileno($ob) . ")" if isdbg 'chan';
277         
278         my $ser = Mojo::IOLoop::Stream->new($ob);
279         $ser->on(error=>sub {dbg "serial $_[1]"; do_reopen($name) unless $ending});
280         $ser->on(close=>sub {dbg "serial closing"; do_reopen($name) unless $ending});
281         $ser->on(timeout=>sub {dbg "serial timeout";});
282         $ser->on(read=>sub {on_read(@_)});
283         $ser->start;
284
285         Mojo::IOLoop->remove($tid) if $tid;
286         undef $tid;
287         Mojo::IOLoop->remove($rid) if $rid;
288         undef $rid;
289         $rid = Mojo::IOLoop->recurring($poll_interval => sub {
290                                                                            start_loop() if !$state;
291                                                                    });
292         chgstate('');
293         $nlcount = 0;
294         
295         return $ser;
296 }
297
298 my @min;
299 my @hour;
300
301 sub process
302 {
303         my $blk = shift;
304         my $loo =  substr $blk,0,3;
305         unless ( $loo eq 'LOO') {
306                 dbg "Block invalid loo -> $loo" if isdbg 'chan'; return;
307         }
308
309         my $tmp;
310         my $temp;
311         my $rain;
312         my %h;
313
314         my $crc_calc = CRC_CCITT($blk);
315
316         if ($crc_calc==0) {
317
318                 # Common ones
319                 $tmp = unpack("s", substr $blk,7,2) / 1000;
320                 $h{Pressure} = nearest(1, in2mb($tmp));
321
322                 $tmp = unpack("s", substr $blk,9,2) / 10;
323                 $h{Temp_In} = nearest(0.1, f2c($tmp));
324
325                 $temp  = nearest(0.1, f2c(unpack("s", substr $blk,12,2) / 10));
326                 $h{Temp_Out}  = $temp;
327                 if ($temp > 75 || $temp < -75) {
328                         dbg "LOOP Temperature out of range ($temp), record ignored";
329                         return;
330                 }
331
332                 $tmp = unpack("C", substr $blk,14,1);
333                 $h{Wind} = nearest(0.1, mph2mps($tmp));
334                 $h{Dir}     = unpack("s", substr $blk,16,2)+0;
335
336                 my $wind = {w => $h{Wind}, d => $h{Dir}};
337                 $wind = 0 if $wind == 255;
338                 push @{$ld->{wind_min}}, $wind;
339
340                 $tmp = int(unpack("C", substr $blk,33,1)+0);
341                 if ($tmp > 100) {
342                         dbg "LOOP Outside Humidity out of range ($tmp), record ignored";
343                         return;
344                 }
345                 $h{Humidity_Out} = $tmp;
346                 $tmp = int(unpack("C", substr $blk,11,1)+0);
347                 if ($tmp > 100) {
348                         dbg "LOOP Inside Humidity out of range ($tmp), record ignored";
349                         return;
350                 }
351                 $h{Humidity_In}  = $tmp;
352                 
353
354                 $tmp = unpack("C", substr $blk,43,1)+0;
355                 $h{UV} = $tmp unless $tmp >= 255;
356                 $tmp = unpack("s", substr $blk,44,2)+0; # watt/m**2
357                 $h{Solar} = $tmp unless $tmp >= 32767;
358
359                 #       $h{Rain_Rate}  = nearest(0.1,unpack("s", substr $blk,41,2) * $rain_mult);
360                 $rain = $h{Rain_Day}   = nearest(0.1, unpack("s", substr $blk,50,2) * $rain_mult);
361                 my $delta_rain = $h{Rain} = nearest(0.1, ($rain >= $ld->{last_rain} ? $rain - $ld->{last_rain} : $rain)) if $loop_count;
362                 $ld->{last_rain} = $rain;
363
364                 # what sort of packet is it?
365                 my $sort =  unpack("C", substr $blk,4,1);
366                 if ($sort) {
367
368                         # Newer LOOP2 packet
369                         $tmp = unpack("C", substr $blk,18,2);
370                         #               $h{Wind_Avg_10} = nearest(0.1,mph2mps($tmp/10));
371                         $tmp = unpack("C", substr $blk,20,2);
372                         #               $h{Wind_Avg_2} = nearest(0.1,mph2mps($tmp/10));
373                         $tmp = unpack("C", substr $blk,22,2);
374                         #               $h{Wind_Gust_10} = nearest(0.1,mph2mps($tmp/10));
375
376                         #               $h{Dir_Avg_10} = unpack("C", substr $blk,24,2)+0;
377                         $tmp = unpack("C", substr $blk,30,2);
378                         $h{Dew_Point} = nearest(0.1, f2c($tmp));
379
380                 } else {
381
382                         # Older LOOP packet
383                         $tmp = unpack("C", substr $blk,15,1);
384                         #               $h{Wind_Avg_10} = nearest(0.1,mph2mps($tmp));
385                         $h{Dew_Point}  = nearest(0.1, dew_point($h{Temp_Out}, $h{Humidity_Out}));
386                         $h{Rain_Month}  = nearest(0.1, unpack("s", substr $blk,52,2) * $rain_mult);
387                         $h{Rain_Year}  = nearest(0.1, unpack("s", substr $blk,54,2) * $rain_mult);
388                 }
389
390                 my $ts = time;
391                 my $s;
392                 my $dayno = int($ts/86400);
393                 if ($dayno > $ld->{last_day}) {
394                         $ld->{Temp_Out_Max} = $ld->{Temp_Out_Min} = $temp;
395                         $ld->{Temp_Out_Max_T} = $ld->{Temp_Out_Min_T} = clocktime($ts, 0);
396                         $ld->{last_day} = $dayno;
397                         write_ld();
398                         cycle_loop_data_files();
399                 }
400                 if ($temp > $ld->{Temp_Out_Max}) {
401                         $ld->{Temp_Out_Max} = $temp;
402                         $ld->{Temp_Out_Max_T} = clocktime($ts, 0);
403                         write_ld();
404                 }
405                 if ($temp < $ld->{Temp_Out_Min}) {
406                         $ld->{Temp_Out_Min} = $temp;
407                         $ld->{Temp_Out_Min_T} = clocktime($ts, 0);
408                         write_ld();
409                 }
410
411                 if ($ts >= $ld->{last_hour} + 1800) {
412                         $h{Pressure_Trend}    = unpack("C", substr $blk,3,1);
413                         $h{Pressure_Trend_txt} = $bar_trend{$h{Pressure_Trend}};
414                         $h{Batt_TX_OK}  = (unpack("C", substr $blk,86,1)+0) ^ 1;
415                         $h{Batt_Console}  = nearest(0.01, unpack("s", substr $blk,87,2) * 0.005859375);
416                         $h{Forecast_Icon}  = unpack("C", substr $blk,89,1);
417                         $h{Forecast_Rule}  = unpack("C", substr $blk,90,1);
418                         $h{Sunrise}  = sprintf( "%04d", unpack("S", substr $blk,91,2) );
419                         $h{Sunrise}  =~ s/(\d{2})(\d{2})/$1:$2/;
420                         $h{Sunset}   = sprintf( "%04d", unpack("S", substr $blk,93,2) );
421                         $h{Sunset}  =~ s/(\d{2})(\d{2})/$1:$2/;
422                         $h{Temp_Out_Max} = $ld->{Temp_Out_Max};
423                         $h{Temp_Out_Min} = $ld->{Temp_Out_Min};
424                         $h{Temp_Out_Max_T} = $ld->{Temp_Out_Max_T};
425                         $h{Temp_Out_Min_T} = $ld->{Temp_Out_Min_T};
426                         
427
428                         if ($loop_count) {      # i.e not the first
429                                 my $a = wind_average(scalar @{$ld->{wind_hour}} ? @{$ld->{wind_hour}} : {w => $h{Wind}, d => $h{Dir}});
430
431                                 $h{Wind_1h} = nearest(0.1, $a->{w});
432                                 $h{Dir_1h} = nearest(0.1, $a->{d});
433
434                                 $a = wind_average(@{$ld->{wind_min}});
435                                 $h{Wind_1m} = nearest(0.1, $a->{w});
436                                 $h{Dir_1m} = nearest(1, $a->{d});
437
438                                 ($h{Rain_1m}, $h{Rain_1h}, $h{Rain_24h}) = calc_rain($rain);
439                         }
440                         $ld->{last_rain_min} = $ld->{last_rain_hour} = $rain;
441
442                         $last_hour_h = {%h};
443                         $s = genstr($ts, 'h', \%h);
444                         $ld->{lasthour_h} = $s;
445                         
446                         $ld->{last_hour} = int($ts/1800)*1800;
447                         $ld->{last_min} = int($ts/60)*60;
448                         @{$ld->{wind_hour}} = ();
449                         @{$ld->{wind_min}} = ();
450
451                         if ($s) {
452                                 output_str($s, 1);
453                                 push @last5daysh, $s;
454                                 shift @last5daysh if @last5daysh > 5*24;
455                         }
456                         write_ld();
457                         
458                 } elsif ($ts >= $ld->{last_min} + 60) {
459                         my $a = wind_average(@{$ld->{wind_min}});
460                         my %save;
461
462                         push @{$ld->{wind_hour}}, $a;
463
464                         if ($loop_count) {      # i.e not the first
465                                 my $rm;
466                                 
467                                 $h{Wind_1m} = nearest(0.1, $a->{w});
468                                 $h{Dir_1m} = nearest(1, $a->{d});
469                                 ($h{Rain_1m}, $h{Rain_1h}, $h{Rain_24h}) = calc_rain($rain);
470                         }
471                         $ld->{last_rain_min} = $rain;
472
473                         $h{Temp_Out_Max} = $ld->{Temp_Out_Max};
474                         $h{Temp_Out_Min} = $ld->{Temp_Out_Min};
475                         $h{Temp_Out_Max_T} = $ld->{Temp_Out_Max_T};
476                         $h{Temp_Out_Min_T} = $ld->{Temp_Out_Min_T};
477
478                         $last_min_h = {%h};
479                         $s = genstr($ts, 'm', \%h);
480                         $ld->{lastmin_h} = $s;
481                         
482                         $ld->{last_min} = int($ts/60)*60;
483                         @{$ld->{wind_min}} = ();
484                         
485                         output_str($s, 1) if $s;
486                         write_ld();
487
488                 } else {
489                         my $o = gen_hash_diff($ld->{last_h}, \%h);
490                         if ($o) {
491                                 $o->{Dir} ||= $h{Dir};
492                                 $o->{Wind} ||= $h{Wind};
493                                 $o->{Dir} += 0;
494                                 $o->{Wind} += 0;
495                                 $s = genstr($ts, 'r', $o);
496                                 push @last10minsr, $s;
497                                 shift @last10minsr while @last10minsr > ($windmins * $updatepermin);
498                         }
499                         else {
500                                 dbg "loop rec not changed" if isdbg 'chan';
501                         }
502                         output_str($s, 0) if $s;
503                 }
504                 $ld->{last_h} = \%h;
505                 ++$loop_count;
506         } else {
507                 dbg "CRC check failed for LOOP data!";
508         }
509         return;
510 }
511
512 sub genstr
513 {
514         my $ts = shift;
515         my $let = shift;
516         my $h = shift;
517         
518         my $j =  $json->encode($h);
519         my $tm = clocktime($ts, 1);
520         return qq|{"tm":"$tm","t":$ts,"$let":$j}|;
521 }
522
523 sub clocktime
524 {
525         my $ts = shift;
526         my $secsreq = shift;
527         my ($sec,$min,$hr) = (gmtime $ts)[0,1,2];
528         my $s;
529         if ($secsreq) {
530                 $s = sprintf "%02d:%02d:%02d", $hr, $min, $sec;
531         } else {
532                 $s = sprintf "%02d:%02d", $hr, $min;
533         }
534         return $s;
535 }
536
537 sub output_str
538 {
539         my $s = shift;
540         my $logit = shift;
541         
542         dbg $s;
543 #       say $s;
544         $dlog->writenow($s) if $logit;
545         foreach my $ws (keys $WS) {
546                 my $tx = $WS->{$ws};
547                 if ($tx) {
548                         $tx->send($s);
549                 } else {
550                         delete $WS->{$tx};
551                 }
552         }
553 }
554
555 sub gen_hash_diff
556 {
557         my $last = shift;
558         my $now = shift;
559         my %o;
560         my $count;
561
562         while (my ($k, $v) = each %$now) {
563                 if (!exists $last->{$k} || $last->{$k} ne $now->{$k}) {
564                         $o{$k} = $v;
565                         ++$count;
566                 }
567         }
568         return $count ? \%o : undef;
569 }
570
571 sub dew_point
572 {
573         my $temp = shift @_;
574         my $rh   = shift @_;
575
576         #  Using the simplified approximation for dew point
577         #  Accurate to 1 degree C for humidities > 50 %
578         #  http://en.wikipedia.org/wiki/Dew_point
579
580         my $dewpoint = $temp - ((100 - $rh) / 5);
581
582         # this is the more complete one (which doesn't work)
583         #my $a = 6.1121;
584         #my $b = 18.678;
585         #my $c = 257.14;
586         #my $ytrh = log(($rh/100) + ($b * $temp) / ($c + $temp));
587         #my $dewpoint = ($c * $ytrh) / ($b - $ytrh); 
588
589         return $dewpoint;
590 }
591
592 sub CRC_CCITT
593 {
594     # Expects packed data...
595     my $data_str = shift @_;
596
597         my $crc = 0;
598         my @lst = split //, $data_str;
599         foreach my $data (@lst) {
600                 my $data = unpack("c",$data);
601
602                 my $crc_prev = $crc;
603                 my $index = $crc >> 8 ^ $data;
604                 my $lhs = $crc_table[$index];
605                 #print "lhs=$lhs, crc=$crc\n";
606                 my $rhs = ($crc << 8) & 0xFFFF;
607                 $crc = $lhs ^ $rhs;
608
609
610         }
611
612         return $crc;
613 }
614
615 sub f2c
616 {
617         return ($_[0] - 32) * 5/9;
618 }
619
620 sub mph2mps
621 {
622         return $_[0] * 0.44704;
623 }
624
625 sub in2mb
626 {
627         return $_[0] * 33.8637526;
628 }
629
630 sub wind_average
631 {
632         my ($sindir, $cosdir, $wind);
633         my $count;
634         
635         foreach my $r (@_) {
636                 $wind += $r->{w};
637                 $sindir += sin(d2r($r->{d})) * $r->{w};
638                 $cosdir += cos(d2r($r->{d})) * $r->{w};
639                 ++$count;
640         }
641
642         my $avhdg = r2d(atan2($sindir, $cosdir));
643         $avhdg += 360 if $avhdg < 0; 
644         return {w => nearest(0.1,$wind / $count), d => nearest(0.1,$avhdg)};
645 }
646
647 # radians to degrees
648 sub r2d
649 {
650     my $n = shift;
651     return ($n / pi) * 180;
652 }
653
654 # degrees to radians
655 sub d2r 
656 {
657     my $n = shift;
658     return ($n / 180) * pi;
659 }
660
661 sub calc_rain
662 {
663         my $rain = shift;
664         
665         $ld->{rain24} ||= [];
666         
667         my $Rain_1h = nearest(0.1, $rain >= $ld->{last_rain_hour} ? $rain - $ld->{last_rain_hour} : $rain); # this is the rate for this hour, so far
668         my $rm = nearest(0.1, $rain >= $ld->{last_rain_min} ? $rain - $ld->{last_rain_min} : $rain);
669         my $Rain_1m = nearest(0.1, $rm);
670         push @{$ld->{rain24}}, $Rain_1m;
671         $ld->{rain_24} += $rm;
672         while (@{$ld->{rain24}} > 24*60) {
673                 $ld->{rain_24} -= shift @{$ld->{rain24}};
674         }
675         my $Rain_24h = nearest(0.1, $ld->{rain_24});
676         return ($Rain_1m, $Rain_1h, $Rain_24h);
677 }
678
679 sub read_ld
680 {
681         unless ($dataf) {
682                 $dataf = IO::File->new("+>> $datafn") or die "cannot open $datafn $!";
683                 $dataf->autoflush(1);
684         }
685
686         seek $dataf, 0, 0;
687         my $s = <$dataf>;
688         chomp $s;
689         dbg "read loop data: $s" if isdbg 'json';
690         $ld = $json->decode($s) if length $s;
691         
692         # sort out rain stats
693         my $c;
694         if ($ld->{rain24} && ($c = @{$ld->{rain24}}) < 24*60) {
695                 my $diff = 24*60 - $c;
696                 unshift @{$ld->{rain24}}, 0 for 0 .. $diff;  
697         }
698         my $rain;
699         
700         if ($ld->{rain24}) {
701                 $rain += $_ for @{$ld->{rain24}};
702         }
703         
704         $ld->{rain_24} = nearest(0.1, $rain);
705         delete $ld->{hour};
706         delete $ld->{min};
707 }
708
709 sub write_ld
710 {
711         unless ($dataf) {
712                 $dataf = IO::File->new("+>> $datafn") or die "cannot open $datafn $!";
713                 $dataf->autoflush(1);
714         }
715
716         seek $dataf, 0, 0;
717         truncate $dataf, 0;
718         $ld->{ts} = time;
719         my $s = $json->encode($ld);
720         dbg "write loop data: $s" if isdbg 'json';
721         print $dataf "$s\n";
722 }
723
724 sub cycle_loop_data_files
725 {
726         $dataf->close if $dataf;
727         undef $dataf;
728         
729         rename "$datafn.oooo", "$datafn.ooooo";
730         rename "$datafn.ooo", "$datafn.oooo";
731         rename "$datafn.oo", "$datafn.ooo";
732         rename "$datafn.o", "$datafn.oo";
733         copy $datafn, "$datafn.o";
734 }
735
736 sub grab_history
737 {
738         my $lg = shift;
739         my $let = shift;
740         my $start = shift || time - 86400;
741         my $dayno = shift;
742         my @out;
743         
744         if ($lg->open($dayno, 'r+')) {
745                 while (my $l = $lg->read) {
746                         next unless $l =~ /,"$let":/;
747                         my ($t) = $l =~ /"t":(\d+)/;
748                         if ($t && $t >= $start) {
749                                 push @out, $l;
750                         }
751                 }
752                 $lg->close;
753         }
754         return @out;
755 }