From: minima Date: Sun, 13 Oct 2002 11:16:08 +0000 (+0000) Subject: added first tranche of USDB stuff X-Git-Tag: PRE-1-52~158 X-Git-Url: http://dxcluster.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=08912ec52dee25bbe00aef10387e1822dcd574bc;p=spider.git added first tranche of USDB stuff --- diff --git a/perl/USDB.pm b/perl/USDB.pm new file mode 100644 index 00000000..3b62fa5d --- /dev/null +++ b/perl/USDB.pm @@ -0,0 +1,113 @@ +# +# Package to handle US Callsign -> City, State translations +# +# Copyright (c) 2002 Dirk Koopman G1TLH +# +# + +use strict; + +use DXVars; +use DB_File; +use File::Copy; +use DXDebug; +use Compress::Zlib; + +use vars qw($VERSION $BRANCH); +$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ ); +$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ || (0,0)); +$main::build += $VERSION; +$main::branch += $BRANCH; + +use vars qw(%db $present); + +my $dbfn = "$main::data/usdb.v1"; + +sub init +{ + end(); + tie %db, 'DB_File', $dbfn and $present = 1; +} + +sub end +{ + return unless $present; + untie %db; + undef $present; +} + +sub get +{ + return () unless $present; + my $ctyn = $db{$_[0]}; + my @s = split /\|/, $db{$ctyn} if $ctyn; + return @s; +} + +sub getstate +{ + return () unless $present; + my @s = get($_[0]); + return @s ? $s[1] : undef; +} + +sub getcity +{ + return () unless $present; + my @s = get($_[0]); + return @s ? $s[0] : undef; +} + +# +# load in / update an existing DB with a standard format (GZIPPED) +# "raw" file. +# +# Note that this removes and overwrites the existing DB file +# You will need to init again after doing this +# + +sub load +{ + # create the new output file + my $a = new DB_File::BTREEINFO; + $a->{psize} = 4096 * 2; + my $s; + if ($s = -s $dbfn && $s > 1024 * 1024) { + $a->{cachesize} = int(($s / (1024*1024)) / 2) * 1024 * 1024; + } + my %dbn; + if (-e $dbfn ) { + syscopy($dbfn, "$dbfn.new") or return "cannot copy $dbfn -> $dbfn.new $!"; + } + + tie %dbn, 'DB_File', "$dbfn.new", O_RDWR|O_CREAT, 0664, $a or return "cannot tie $dbfn.new $!"; + + # now write away all the files + for (@_) { + my $fn = shift; + my $f = gzopen($fn, "r") or return "Cannot open $fn $!"; + while ($f->gzreadline) { + chomp; + my ($call, $city, $state) = split /\|/; + + # lookup the city + my $s = "$city|$state"; + my $ctyn = $dbn{$s}; + unless ($ctyn) { + my $no = $dbn{'##'} || 1; + $ctyn = "#$no"; + $dbn{$s} = $ctyn; + $dbn{$ctyn} = $s; + $no++; + $dbn{'##'} = "$no"; + } + $dbn{$call} = $ctyn; + } + $f->gzclose; + } + + untie %dbn; + rename "$dbfn.new", $dbfn; +} + +1; diff --git a/perl/create_usdb.pl b/perl/create_usdb.pl new file mode 100755 index 00000000..a0da7ff4 --- /dev/null +++ b/perl/create_usdb.pl @@ -0,0 +1,36 @@ +#!/usr/bin/perl +# +# create a USDB file from a standard raw file (which is GZIPPED BTW) +# +# This will overwrite and remove any existing usdb file, but it will +# copy the old one first and update that. +# + +use strict; + +# make sure that modules are searched in the order local then perl +BEGIN { + # root of directory tree for this system + use vars qw($root); + + $root = "/spider"; + $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; + + unshift @INC, "$root/local"; +} + +use vars qw($VERSION $BRANCH); +$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ ); +$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ || (0,0)); +$main::build += $VERSION; +$main::branch += $BRANCH; + +use DXVars; +use USDB; + +die "no input (usdbraw?) files specified\n" unless @ARGV; + +USDB::load(@ARGV); +exit(0); + + diff --git a/perl/gen_usdb_data.pl b/perl/gen_usdb_data.pl new file mode 100755 index 00000000..153a5601 --- /dev/null +++ b/perl/gen_usdb_data.pl @@ -0,0 +1,130 @@ +#!/usr/bin/perl +# +# Something to create my subset of the US call book data, +# in my flat file form, either from the main data base or +# else the daily updates. +# +# You can get the main database from: +# +# http://wireless.fcc.gov/uls/data/complete/l_amat.zip +# +# The daily data bases are available as a set of seven from here:- +# +# http://wireless.fcc.gov/uls/data/daily/l_am_sat.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_sun.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_mon.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_tue.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_wed.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_thu.zip +# http://wireless.fcc.gov/uls/data/daily/l_am_fri.zip +# +# this program expects one or more zip files containing the call book +# data as arguments. +# +# Copyright (c) 2002 Dirk Koopman G1TLH +# +# $Id$ +# + +use strict; + +# make sure that modules are searched in the order local then perl +BEGIN { + # root of directory tree for this system + use vars qw($root); + + $root = "/spider"; + $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; + + unshift @INC, "$root/local"; +} + +use vars qw($VERSION $BRANCH); +$VERSION = sprintf( "%d.%03d", q$Revision$ =~ /(\d+)\.(\d+)/ ); +$BRANCH = sprintf( "%d.%03d", q$Revision$ =~ /\d+\.\d+\.(\d+)\.(\d+)/ || (0,0)); +$main::build += $VERSION; +$main::branch += $BRANCH; + +use DXVars; +use Archive::Zip qw(:ERROR_CODES); +use Archive::Zip::MemberRead; +use IO::File; +use Compress::Zlib; + +my $blksize = 1024 * 1024; + +STDOUT->autoflush(1); + +my $dbrawfn = "$main::data/usdbraw"; + +rename "$dbrawfn.oo", "$dbrawfn.ooo"; +rename "$dbrawfn.o", "$dbrawfn.oo"; +rename "$dbrawfn", "$dbrawfn.o"; +my $gzfh = gzopen($dbrawfn, "wb9") or die "Cannot open $dbrawfn $!"; + +my $ctycount; + +foreach my $argv (@ARGV) { + my $zip = new Archive::Zip($argv) or die "Cannot open $argv $!\n"; + print "Doing $argv\n"; + handleEN($zip, $argv); + handleAM($zip, $argv); + handleHS($zip, $argv); +} + +$gzfh->gzclose; +print "$ctycount Cities found\n"; + +exit(0); + +sub handleEN +{ + my ($zip, $argv) = @_; + my $mname = "EN.dat"; + my $ofn = "$main::data/$mname"; + print " Handling EN records, unzipping"; + if ($zip->extractMember($mname, $ofn) == AZ_OK) { + my $fh = new IO::File "$ofn" or die "Cannot open $ofn $!"; + if ($fh) { + + print ", storing"; + + my $count; + my $buf; + + while (my $l = $fh->getline) { + $l =~ s/[\r\n]+$//; + my ($rt,$usi,$ulsfn,$ebfno,$call,$type,$lid,$name,$first,$middle,$last,$suffix, + $phone,$fax,$email,$street,$city,$state,$zip,$pobox,$attl,$sgin,$frn) = split /\|/, $l; + + my $rec = uc join '|', $call,$city,$state if $city && $state; + $buf .= "$rec\n"; + if (length $buf > $blksize) { + $gzfh->gzwrite($buf); + undef $buf; + } + my $c = uc "$city|$state"; + $count++; + } + if (length $buf > $blksize) { + $gzfh->gzwrite($buf); + } + print ", $count records\n"; + $fh->close; + } + unlink $ofn; + } else { + print "EN missing in $argv\n"; + return; + } +} + +sub handleAM +{ + +} + +sub handleHS +{ + +}