use vars qw(%dup $duplth $dupage $filterdef);
$duplth = 60; # the length of text to use in the deduping
use vars qw(%dup $duplth $dupage $filterdef);
$duplth = 60; # the length of text to use in the deduping
$filterdef = bless ([
# tag, sort, field, priv, special parser
['by', 'c', 0],
$filterdef = bless ([
# tag, sort, field, priv, special parser
['by', 'c', 0],
unpad($text);
$text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
unpad($text);
$text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
$text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1);
$text =~ s/[^\#a-zA-Z0-9]//g;
$text = substr($text, 0, $duplth) if length $text > $duplth;
$text =~ s/[^\#a-zA-Z0-9]//g;
$text = substr($text, 0, $duplth) if length $text > $duplth;