X-Git-Url: http://dxcluster.org/gitweb/gitweb.cgi?a=blobdiff_plain;f=perl%2FBadWords.pm;h=36db8ffba44384d76a3bc8acbc9789d0447cbbef;hb=888290f339e2ee00894445fecb14f0b506d12368;hp=e7d1169e3a390abd5845f5922bb299eaa79cefdc;hpb=17f0b57add792391822d38116e89b33c1df4e2dd;p=spider.git

diff --git a/perl/BadWords.pm b/perl/BadWords.pm
index e7d1169e..36db8ffb 100644
--- a/perl/BadWords.pm
+++ b/perl/BadWords.pm
@@ -13,11 +13,20 @@ use strict;
 use DXUtil;
 use DXVars;
 use DXHash;
+use DXDebug;
+
 use IO::File;
 
-use vars qw($badword);
+use vars qw($badword @regex);
 
 my $oldfn = "$main::data/badwords";
+my $regex = "$main::data/badw_regex";
+my $bwfn = "$main::data/badword";
+
+# copy issue ones across
+filecopy("$regex.issue", $regex) unless -e $regex;
+filecopy("$bwfn.issue", $bwfn) unless -e $bwfn;
+
 $badword = new DXHash "badword";
 
 use vars qw($VERSION $BRANCH);
@@ -30,7 +39,6 @@ $main::branch += $BRANCH;
 sub load
 {
 	my @out;
-	return unless -e $oldfn;
 	my $fh = new IO::File $oldfn;
 	
 	if ($fh) {
@@ -45,18 +53,64 @@ sub load
 		$fh->close;
 		$badword->put;
 		unlink $oldfn;
+	}
+	push @out, create_regex(); 
+	return @out;
+}
+
+sub create_regex
+{
+	my @out;
+	@regex = ();
+	
+	my $fh = new IO::File $regex;
+	
+	if ($fh) {
+		while (<$fh>) {
+			chomp;
+			next if /^\s*\#/;
+			my @list = split " ";
+			for (@list) {
+				# create a closure for each word so that it matches stuff with spaces/punctuation
+				# and repeated characters in it
+				my $w = uc $_;
+				my @l = map { $_ eq 'I' ? '[I1]' : ($_ eq 'O' ? '[O0]' : $_) }split //, $w;
+				my $e = join '+[\s\W]+', @l;
+				my $s = eval qq{sub { return \$_[0] =~ /$e+/ ? '$w' : () } };
+				push @regex, $s unless $@;
+				dbg("create_regex: $@") if $@;
+			}
+		}
+		$fh->close;
 	} else {
-		my $l = "can't open $oldfn $!";
+		my $l = "can't open $regex $!";
 		dbg($l);
 		push @out, $l;
 	}
+	
 	return @out;
 }
 
 # check the text against the badwords list
 sub check
 {
-	return grep { $badword->in($_) } split(/\b/, lc shift);
+	my $s = uc shift;
+	my @out;
+	
+	for (@regex) {
+		push @out, &$_($s);
+	}
+	
+	return @out if @out;
+	
+	for (split(/\s+/, $s)) {
+		s/[^\w]//g;
+		push @out, $_ if $badword->in($_);
+		s/\'?S$//;
+		push @out, $_ if $badword->in($_);
+	}
+
+	return @out;
 }
 
 1;