2 # DX cluster user routines
4 # Copyright (c) 1998-2020 - Dirk Koopman G1TLH
6 # The new internal structure of the users system looks like this:
8 # The users.v4 file formatted as a file of lines containing: <callsign>\t{json serialised version of user record}\n
10 # You can look at it with any text tools or your favourite editor :-)
12 # In terms of internal structure, the main user hash remains as %u, keyed on callsign as before.
14 # The value is a one or two element array [position] or [position, ref], depending on whether the record has been "get()ed"
15 # [i.e. got from disk] or not. The 'position' is simply the start of each line in the file. The function "get()" simply returns
16 # the stored reference in array[1], if present, or seeks to the position from array[0], reads a line, json_decodes it,
17 # stores that reference into array[1] and returns that. That reference will be used from that time onwards.
19 # The routine writeoutjson() will (very) lazily write out a copy of %u WITHOUT STORING ANY EXTRA CURRENTLY UNREFERENCED CALLSIGN
20 # records to users.v4.n. It, in effect, does a sort of random accessed merge of the current user file and any "in memory"
21 # versions of any user record. This can be done with a spawned command because it will just be reading %u and merging
22 # loaded records, not altering the current users.v4 file in any way.
24 # %u -> $u{call} -> [position of json line in users.v4 (, reference -> {call=>'G1TLH', ...} if this record is in use)].
26 # On my machine, it takes about 250mS to read the entire users.v4 file of 190,000 records and to create a
27 # $u{callsign}->[record position in users.v4] for every callsign in the users.v4 file. Loading ~19,000 records
28 # (read from disk, decode json, store reference) takes about 110mS (or 580nS/record).
30 # A periodic dump of users.v4.n, with said ~19,000 records in memory takes about 750mS to write (this can be speeded up,
31 # by at least a half, if it becomes a problem!). As this periodic dump will be spawned off, it will not interrupt the data
34 # This is the first rewrite of DXUsers since inception. In the mojo branch we will no longer use Storable but use JSON instead.
35 # We will now be storing all the keys in memory and will use opportunistic loading of actual records in "get()". So out of
36 # say 200,000 known users it is unlikely that we will have more than 10% (more likely less) of the user records in memory.
37 # This will mean that there will be a increase in memory requirement, but it is modest. I estimate it's unlikely be more
40 # At the moment that means that the working users.v4 is "immutable".
42 # In normal operation, when first calling 'init()', the keys and positions will be read from the newer of users.v4.n and
43 # users.v4. If there is no users.v4.n, then users.v4 will be used. As time wears on, %u will then accrete active user records.
44 # Once an hour the current %u will be saved to users.v4.n.
46 # If it becomes too much of a problem then we are likely to chuck off "close()d" users onto the end of the current users.v4
47 # leaving existing users intact, but updating the pointer to the (now cleared out) user ref to the new location. This will
48 # be a sort of write behind log file. The users.v4 file is still immutable for the starting positions, but any chucked off
49 # records (or even "updates") will be written to the end of that file. If this has to be reread at any time, then the last
50 # entry for any callsign "wins". But this will only happen if I think the memory requirements over time become too much.
52 # As there is no functional difference between the users.v4 and export_user generated "user_json" file(s), other than the latter
53 # will be in sorted order with the record elements in "canonical" order. There will now longer be any code to execute to
54 # "restore the users file". Simply copy one of the "user_json" files to users.v4, remove users.v4.n and restart.
56 # Hopefully though, this will put to rest the need to do all that messing about ever again... Pigs may well be seen flying over
57 # your node as well :-)
72 use Data::Structure::Util qw(unbless);
73 use Time::HiRes qw(gettimeofday tv_interval);
78 use vars qw(%u $dbm $filename %valid $lastoperinterval $lasttime $lru $lrusize $tooold $v3 $v4);
83 $lastoperinterval = 60*24*60*60;
86 $tooold = 86400 * 365 + 31; # this marks an old user who hasn't given enough info to be useful
91 our $maxconnlist = 3; # remember this many connection time (duration) [start, end] pairs
93 our $newusers; # per execution stats
98 my $ifh; # the input file, initialised by readinjson()
101 # hash of valid elements and a simple prompt
103 call => '0,Callsign',
104 alias => '0,Real Callsign',
107 lat => '0,Latitude,slat',
108 long => '0,Longitude,slong',
110 email => '0,E-mail Address,parray',
111 priv => '9,Privilege Level',
112 lastin => '0,Last Time in,cldatetime',
113 passwd => '9,Password,yesno',
114 passphrase => '9,Pass Phrase,yesno',
115 addr => '0,Full Address',
116 'sort' => '0,Type of User', # A - ak1a, U - User, S - spider cluster, B - BBS
117 xpert => '0,Expert Status,yesno',
119 node => '0,Last Node',
120 homenode => '0,Home Node',
121 lockout => '9,Locked out?,yesno', # won't let them in at all
122 dxok => '9,Accept DX Spots?,yesno', # accept his dx spots?
123 annok => '9,Accept Announces?,yesno', # accept his announces?
124 lang => '0,Language',
125 hmsgno => '0,Highest Msgno',
126 group => '0,Group,parray', # used to create a group of users/nodes for some purpose or other
127 buddies => '0,Buddies,parray',
128 isolate => '9,Isolate network,yesno',
129 wantbeep => '0,Req Beep,yesno',
130 wantann => '0,Req Announce,yesno',
131 wantwwv => '0,Req WWV,yesno',
132 wantwcy => '0,Req WCY,yesno',
133 wantecho => '0,Req Echo,yesno',
134 wanttalk => '0,Req Talk,yesno',
135 wantwx => '0,Req WX,yesno',
136 wantdx => '0,Req DX Spots,yesno',
137 wantemail => '0,Req Msgs as Email,yesno',
138 pagelth => '0,Current Pagelth',
139 pingint => '9,Node Ping interval',
140 nopings => '9,Ping Obs Count',
141 wantlogininfo => '0,Login Info Req,yesno',
142 wantgrid => '0,Show DX Grid,yesno',
143 wantann_talk => '0,Talklike Anns,yesno',
144 wantpc16 => '9,Want Users from node,yesno',
145 wantsendpc16 => '9,Send PC16,yesno',
146 wantroutepc19 => '9,Route PC19,yesno',
147 wantusstate => '0,Show US State,yesno',
148 wantdxcq => '0,Show CQ Zone,yesno',
149 wantdxitu => '0,Show ITU Zone,yesno',
150 wantgtk => '0,Want GTK interface,yesno',
151 wantpc9x => '0,Want PC9X interface,yesno',
152 lastoper => '9,Last for/oper,cldatetime',
153 nothere => '0,Not Here Text',
154 registered => '9,Registered?,yesno',
155 prompt => '0,Required Prompt',
156 version => '1,Version',
158 believe => '1,Believable nodes,parray',
159 lastping => '1,Last Ping at,ptimelist',
160 maxconnect => '1,Max Connections',
161 startt => '0,Start Time,cldatetime',
162 connlist => '1,Connections,parraydifft',
169 my $name = $AUTOLOAD;
171 return if $name =~ /::DESTROY$/;
174 confess "Non-existant field '$AUTOLOAD'" if !$valid{$name};
175 # this clever line of code creates a subroutine which takes over from autoload
176 # from OO Perl - Conway
177 *$AUTOLOAD = sub {@_ > 1 ? $_[0]->{$name} = $_[1] : $_[0]->{$name}};
184 # initialise the system
195 $json = JSON->new()->canonical(1);
196 $filename = $ufn = localdata("$fn.json");
198 if (-e localdata("$fn.json")) {
206 if ( ! -e localdata("users.v3") && -e localdata("users.v2") ) {
209 LogDbg('',"the module Storable appears to be missing!!");
210 LogDbg('',"trying to continue in compatibility mode (this may fail)");
211 LogDbg('',"please install Storable from CPAN as soon as possible");
213 import Storable qw(nfreeze thaw);
214 $convert = 3 if -e localdata("users.v3") && !-e $ufn;
218 # do a conversion if required
220 my ($key, $val, $action, $count, $err) = ('','',0,0,0);
221 my $ta = [gettimeofday];
224 LogDbg('',"Converting the User File from V$convert to $fn.json ");
225 LogDbg('',"This will take a while, I suggest you go and have cup of strong tea");
226 my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]";
227 for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) {
230 eval { $ref = storable_decode($val) };
232 eval { $ref = asc_decode($val) };
242 Log('err', "DXUser: error decoding $@");
247 my $t = _diffms($ta);
248 LogDbg('',"Conversion from users.v$convert to users.json completed $count records $err errors $t mS");
250 # now write it away for future use
251 $ta = [gettimeofday];
253 $count = writeoutjson();
255 LogDbg('',"New Userfile users.json write completed $count records $err errors $t mS");
256 LogDbg('',"Now restarting..");
259 # otherwise (i.e normally) slurp it in
267 # with extreme prejudice
269 unlink "$main::data/users.v3";
270 unlink "$main::local_data/users.v3";
273 unlink "$main::data/users.v4";
274 unlink "$main::local_data/users.v4";
279 # periodic processing
283 # if ($main::systime > $lasttime + 15) {
284 # #$dbm->sync if $dbm;
285 # $lasttime = $main::systime;
300 # new - create a new user
307 my $self = bless {call => $call, 'sort'=>'U'}, $pkg;
315 # $call =~ s/-\d+$//o;
317 # confess "can't create existing call $call in User\n!" if $u{$call};
319 my $self = $pkg->alloc($call);
327 # get - get an existing user - this seems to return a different reference everytime it is
334 my $nodecode = shift;
336 return undef unless $ref;
339 $ifh->seek($ref->[0], 0);
340 my $l = $ifh->getline;
342 my ($k,$s) = split /\t/, $l;
343 return $s if $nodecode;
344 my $j = json_decode($s);
349 } elsif ($nodecode) {
350 return json_encode($ref->[1]);
356 # get an "ephemeral" reference - i.e. this will give you new temporary copy of
357 # the call's user record, but without storing it (if it isn't already there)
359 # This is not as quick as get()! But it will allow safe querying of the
360 # user file. Probably in conjunction with get_some_calls feeding it.
362 # Probably need to create a new copy of any existing records WIP
369 $ifh->seek($ref->[0], 0);
370 my $l = $ifh->getline;
372 my ($k,$s) = split /\t/, $l;
373 my $j = json_decode($s);
381 # get an existing either from the channel (if there is one) or from the database
383 # It is important to note that if you have done a get (for the channel say) and you
384 # want access or modify that you must use this call (and you must NOT use get's all
385 # over the place willy nilly!)
392 my $dxchan = DXChannel::get($call);
394 my $ref = $dxchan->user;
395 return $ref if $ref && UNIVERSAL::isa($ref, 'DXUser');
397 dbg("DXUser::get_current: got invalid user ref for $call from dxchan $dxchan->{call} ". ref $ref. " ignoring");
403 # get all callsigns in the database
408 return (sort keys %u);
412 # get some calls - provide a qr// style selector string as a partial key
417 my $pattern = shift || qr/.*/;
418 return sort grep {$pattern} keys %u;
428 confess "Trying to put nothing!" unless $self && ref $self;
429 $self->{lastin} = $main::systime;
430 ++$modusers; # new or existing, it's still been modified
436 goto &json_encode if $v4;
437 goto &asc_encode unless $v3;
439 return nfreeze($self);
445 goto &json_decode if $v4;
446 goto &storable_decode if $v3;
450 # should now be obsolete for mojo branch build 238 and above
460 # create a hash from a string (in ascii)
466 $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
469 LogDbg('err', "DXUser::asc_decode: on '$s' $@");
479 eval { $ref = $json->decode($s) };
481 return bless $ref, 'DXUser';
483 LogDbg('err', "DXUser::json_decode: on '$s' $@");
492 my $s = $json->encode($ref);
493 bless $ref, 'DXUser';
498 # del - delete a user
504 my $call = $self->{call};
505 # $lru->remove($call);
513 # close - close down a user
521 $self->{lastin} = $main::systime;
522 # add a record to the connect list
523 my $ref = [$startt || $self->{startt}, $main::systime];
524 push @$ref, $ip if $ip;
525 push @{$self->{connlist}}, $ref;
526 shift @{$self->{connlist}} if @{$self->{connlist}} > $maxconnlist;
540 # return a list of valid elements
550 # export the database to an ascii file
557 my $fn = $name || localdata("user_json"); # force use of local_data
560 move "$fn.oooo", "$fn.ooooo" if -e "$fn.oooo";
561 move "$fn.ooo", "$fn.oooo" if -e "$fn.ooo";
562 move "$fn.oo", "$fn.ooo" if -e "$fn.oo";
563 move "$fn.o", "$fn.oo" if -e "$fn.o";
564 move "$fn", "$fn.o" if -e "$fn";
566 my $json = JSON->new;
572 my $fh = new IO::File ">$fn" or return "cannot open $fn ($!)";
576 foreach my $k (sort keys %u) {
578 if ($r->{sort} eq 'U' && !$r->{priv} && $main::systime > $r->{lastin}+$tooold ) {
579 unless ($r->{lat} || $r->{long} || $r->{qra} || $r->{qth} || $r->{name}) {
580 LogDbg('err', "DXUser::export deleting $k - too old, last in " . cldatetime($r->lastin) . " " . difft([$r->lastin, $main::systime]));
586 eval {$val = json_encode($r);};
588 LogDbg('err', "DXUser::export error encoding call: $k $@");
592 $fh->print("$k\t$val\n");
597 my $s = qq{Exported users to $fn - $count Users $del Deleted $err Errors ('sh/log Export' for details)};
598 LogDbg('command', $s);
606 # add one or more groups
610 my $ref = $self->{group} || [ 'local' ];
611 $self->{group} = $ref if !$self->{group};
612 push @$ref, @_ if @_;
615 # remove one or more groups
619 my $ref = $self->{group} || [ 'local' ];
622 $self->{group} = $ref if !$self->{group};
624 @$ref = map { my $a = $_; return (grep { $_ eq $a } @in) ? () : $a } @$ref;
627 # does this thing contain all the groups listed?
631 my $ref = $self->{group};
634 return 0 if !$ref || @_ == 0;
635 return 1 if @$ref == 0 && @_ == 0;
636 for ($n = 0; $n < @_; ) {
639 $n++ if grep $_ eq $a, @_;
645 # simplified group test just for one group
650 my $ref = $self->{group};
653 return grep $_ eq $s, $ref;
656 # set up a default group (only happens for them's that connect direct)
660 $self->{group} = [ 'local' ];
663 # set up empty buddies (only happens for them's that connect direct)
667 $self->{buddies} = [ ];
671 # return a prompt for a field
676 my ($self, $ele) = @_;
680 # some variable accessors
684 @_ ? $self->{'sort'} = shift : $self->{'sort'} ;
689 # want is default = 1
696 $self->{$s} = $val if defined $val;
697 return exists $self->{$s} ? $self->{$s} : 1;
700 # wantnot is default = 0
707 $self->{$s} = $val if defined $val;
708 return exists $self->{$s} ? $self->{$s} : 0;
713 return _want('beep', @_);
718 return _want('ann', @_);
723 return _want('wwv', @_);
728 return _want('wcy', @_);
733 return _want('echo', @_);
738 return _want('wx', @_);
743 return _want('dx', @_);
748 return _want('talk', @_);
753 return _want('grid', @_);
758 return _want('email', @_);
763 return _want('ann_talk', @_);
768 return _want('pc16', @_);
773 return _want('sendpc16', @_);
778 return _want('routepc16', @_);
783 return _want('usstate', @_);
788 return _want('dxcq', @_);
793 return _want('dxitu', @_);
798 return _want('gtk', @_);
803 return _want('pc9x', @_);
810 $self->{wantlogininfo} = $val if defined $val;
811 return $self->{wantlogininfo};
817 return $self->{sort} =~ /^[ACRSX]$/;
823 return grep $_ eq 'local_node', @{$self->{group}};
829 return $self->{sort} =~ /^[UW]$/;
835 return $self->{sort} eq 'W';
841 return $self->{sort} eq 'B';
847 return $self->{sort} eq 'S';
853 return $self->{sort} eq 'C';
859 return $self->{sort} eq 'X';
865 return $self->{sort} eq 'R';
871 return $self->{sort} eq 'A';
877 delete $self->{passwd};
883 delete $self->{passphrase};
890 $self->{believe} ||= [];
891 push @{$self->{believe}}, $call unless grep $_ eq $call, @{$self->{believe}};
898 if (exists $self->{believe}) {
899 $self->{believe} = [grep {$_ ne $call} @{$self->{believe}}];
900 delete $self->{believe} unless @{$self->{believe}};
907 return exists $self->{believe} ? @{$self->{believe}} : ();
914 $self->{lastping} ||= {};
915 $self->{lastping} = {} unless ref $self->{lastping};
916 my $b = $self->{lastping};
917 $b->{$call} = shift if @_;
922 # read in the latest version of the user file. As this file is immutable, the file one really wants is
923 # a later (generated) copy. But, if the plain users.v4 file is all we have, we'll use that.
934 my $ta = [gettimeofday];
940 dbg("DXUser $fn not found - probably about to convert");
944 if (-e $nfn && -e $fn && (stat($nfn))[9] > (stat($fn))[9]) {
945 # move the old file to .o
954 $ifh = IO::File->new("+<$fn") or die "$fn read error $!";
956 my $pos = $ifh->tell;
966 # $ifh is "global" and should not be closed
968 dbg("DXUser::readinjson $count record headers read from $fn in ". _diffms($ta) . " mS");
969 return $totusers = $count;
973 # Write a newer copy of the users.v4 file to users.v4.n, which is what will be read in.
974 # This means that the existing users.v4 is not touched during a run of dxspider, or at least
979 my $ofn = shift || "$filename.n";
980 my $ta = [gettimeofday];
982 my $ofh = IO::File->new(">$ofn") or die "$ofn write error $!";
985 for my $k (sort keys %u) {
989 print $ofh "$k\t$l\n";
992 LogDbg('DXCommand', "DXUser::writeoutjson callsign $k not found")
997 dbg("DXUser::writeoutjson $count records written to $ofn in ". _diffms($ta) . " mS");