summaryrefslogtreecommitdiffabout
path: root/TokensV2.pm
authorunknown <gabocze@git.gnu.org.ua>2014-09-23 07:16:06 (GMT)
committer unknown <gabocze@git.gnu.org.ua>2014-09-23 07:16:06 (GMT)
commitfa6b243b08c2e483b8b6ed9654417607f158d416 (patch) (side-by-side diff)
tree80a5e4953d4166f494316d68fb3515282a3bba39 /TokensV2.pm
parent84251691307b7e99bd7cbd15a20190ca14c65a1d (diff)
downloadvital-fa6b243b08c2e483b8b6ed9654417607f158d416.tar.gz
vital-fa6b243b08c2e483b8b6ed9654417607f158d416.tar.bz2
Changed file extensions from txt to {pl|pm}
Diffstat (limited to 'TokensV2.pm') (more/less context) (ignore whitespace changes)
-rw-r--r--TokensV2.pm174
1 files changed, 174 insertions, 0 deletions
diff --git a/TokensV2.pm b/TokensV2.pm
new file mode 100644
index 0000000..619d7df
--- a/dev/null
+++ b/TokensV2.pm
@@ -0,0 +1,174 @@
+=pod
+ TokensV2: Programme to discover tokens, where there are not.
+
+ Copyright 2013 Gabriel Czernikier
+
+
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+=cut
+use strict;
+#package declarations
+sub digest_single;
+sub parse_programme;
+sub printAll;
+
+my @REGEX;
+my @MASK;
+my $long_tokens;
+my @DIGEST_SINGLE;
+my $literal_char_count;
+
+sub digest_single { # $target, $regex, $eaten_left_literal_chars, $eaten_right_literal_chars
+ #return unless $_[1] ne '';
+ #return if length $_[1] < 8; # x>(.*?)<
+ return unless ($_[2]+$_[3])/$literal_char_count<0.999999;
+ return @{$DIGEST_SINGLE[$_[2]][$_[3]]} if defined $DIGEST_SINGLE[$_[2]][$_[3]];
+ pos($_[0]) = undef;
+ goto VISITING unless $_[0] =~ /(${_[1]})/g;
+ my $digit = 2;
+ while( my $cg = eval '$'.$digit++ ) {
+ my @suspicious_tokens = $cg =~ /$long_tokens/g;
+ goto VISITING if @suspicious_tokens>=2;
+ }
+ my $pff = (pos $_[0]);
+ my $pii = $pff - (length $1) if defined $pff;
+ $DIGEST_SINGLE[$_[2]][$_[3]] = [$_[2], $_[3], $pii, $pff];
+ return @{$DIGEST_SINGLE[$_[2]][$_[3]]};
+
+ VISITING:
+ my $re = $_[1];
+ # strip off left shelter, also understood as walking righwards across the regex .+? stopping at a hopefuly serviceable string
+ goto VISITING_2 unless $re =~ s/(.+?)(.\(\.\*\?\).|\([^.])/\2/; # .+? is also the "shelter"
+ my $increment_eaten_literal_chars = $+[1];
+ $increment_eaten_literal_chars -= 5 if $1 =~ /\(\.\*\?\)/; # discount the single occurrence of non-literal chars: (.*?), left alone surrounding ""
+ # expansions
+ $re =~ s/^\(\?:(?:[^)]|\)[^?+*])+\)\*//;
+ $re =~ s/^\(\?:((?:[^)]|\)[^?+*])+)\)\+/\1\(\?:\1\)\*/;
+ $re =~ s/^\(\?:((?:[^)]|\)[^?+*])+)\)\?/\1/;
+ my ($el, $er, $pi, $pf) = digest_single $_[0], $re, $_[2]+$increment_eaten_literal_chars, $_[3];
+
+ VISITING_2:
+ $re = $_[1];
+ # strip off right shelter, also understood as walking leftwards across the regex .+? stopping at a hopefuly serviceable string
+ goto SUBDIGEST unless $re =~ s/(.*(?:.\(\.\*\?\).|\)[?+*]))(.+?)$/\1/; # .+? is the "shelter"
+
+ $increment_eaten_literal_chars = $+[2] - $-[2];
+ $increment_eaten_literal_chars -= 5 if $2 =~ /\(\.\*\?\)/; # discount the single occurrence of non-literal chars: (.*?), left alone surrounding ""
+ # expansions
+ $re =~ s/\(\?:(?:[^)]|\)[^?+*])+\)\*$//;
+ $re =~ s/\(\?:((?:[^)]|\)[^?+*])+)\)\+$/\(\?:\1\)\*\1/;
+ $re =~ s/\(\?:(?:[^)]|\)[^?+*])+\)\?$//;
+ my ($el2, $er2, $pi2, $pf2) = digest_single $_[0], $re, $_[2], $_[3]+$increment_eaten_literal_chars;
+
+ SUBDIGEST:
+ $DIGEST_SINGLE[$_[2]][$_[3]] = [$el, $er, $pi, $pf] if (defined $pi) && ($er+$el<=$er2+$el2 || !defined $pi2);
+ return @{$DIGEST_SINGLE[$_[2]][$_[3]]} if defined $DIGEST_SINGLE[$_[2]][$_[3]];
+ $DIGEST_SINGLE[$_[2]][$_[3]] = [$el2, $er2, $pi2, $pf2] if (defined $pi2) && ($er2+$el2<$er+$el || !defined $pi);
+ return @{$DIGEST_SINGLE[$_[2]][$_[3]]} if defined $DIGEST_SINGLE[$_[2]][$_[3]];
+ $DIGEST_SINGLE[$_[2]][$_[3]] = [];
+ return @{$DIGEST_SINGLE[$_[2]][$_[3]]};
+}
+
+sub digest_multiple {
+ my $regex_num = 0;
+ my $Pos_Ini;
+ my $Eaten_Left;
+ my $Eaten_Right;
+ my $Pos_Fin;
+ my $Regex_Num;
+
+ my @DIGEST_MULTIPLE = ();
+ for my $regex (@REGEX) {
+ @DIGEST_SINGLE = undef;
+ my @literal_chars = $regex =~ /[^.*(?:)+]/g;
+ $literal_char_count = scalar @literal_chars;
+ my ($eaten_left, $eaten_right, $pos_ini, $pos_fin) = digest_single $_[0], $regex, 0, 0;
+ return $eaten_left, $eaten_right, $pos_ini, $pos_fin, $regex_num if(($eaten_right+$eaten_left)/$literal_char_count<0.2);
+ push @DIGEST_MULTIPLE, [$eaten_left, $eaten_right, $pos_ini, $pos_fin, $regex_num] if defined $pos_ini && $pos_fin!=0;
+ $regex_num++;
+ }
+
+ $regex_num = 0;
+ while(@DIGEST_MULTIPLE!=0) {
+ my ($eaten_left, $eaten_right, $pos_ini, $pos_fin) = @{shift @DIGEST_MULTIPLE};
+ if($eaten_right+$eaten_left<$Eaten_Right+$Eaten_Left || !defined $Pos_Ini) {
+ $Pos_Ini = $pos_ini;
+ $Eaten_Left = $eaten_left;
+ $Eaten_Right = $eaten_right;
+ $Pos_Fin = $pos_fin;
+ $Regex_Num = $regex_num;
+ }
+ $regex_num++;
+ }
+ return $Eaten_Left, $Eaten_Right, $Pos_Ini, $Pos_Fin, $Regex_Num if defined $Pos_Ini;
+}
+
+sub parse_programme {
+ return if $_[0] eq '';
+ return $_[0] if length $_[0] < 8; # <x>y</x>
+ my ($eaten_left, $eaten_right, $pos_ini, $pos_fin, $regex_num) = digest_multiple($_[0]);
+ if(not defined $pos_ini) {
+ #my $oldfh = select;
+ #select STDERR;
+ #print "$ARGV[1], [ini-no-parseable]${target}[fin-no-parseable]\n";
+ #select $oldfh;
+ return $_[0];
+ }
+ my $mask_right = @MASK[$regex_num];
+ $mask_right =~ s/.*(.{$eaten_right})/\1/;
+ my $mask_left = @MASK[$regex_num];
+ $mask_left =~ s/(.{$eaten_left}).*/\1/;
+ my $match_length = $pos_fin-$pos_ini;
+ my ($target_left,$match,$target_right) = $_[0] =~ /(.{$pos_ini})(.{$match_length})(.*)/;
+ shift; # discard unused argument
+ return $mask_left.$match.$mask_right, +[parse_programme $target_left], +[parse_programme $target_right], $regex_num;
+}
+
+sub printAll {
+ return if @_==0;
+ if (@_==1) {
+ print "[ini-nonparse]${_[0]}[fin-nonparse]\n";
+ return;
+ }
+ printAll @{$_[1]};
+ print "[ini-prod]${_[0]}[fin-prod]\n";
+ printAll @{$_[2]};
+}
+
+sub parse {
+ @REGEX = ();
+ @MASK = ();
+ my $target = shift;
+ my $regex = shift;
+ while($regex =~ /^(.+)$/mg) {
+ my $_ = $1;
+ my $other = $_;
+ # strip out non-literal chars from MASK
+ $other =~ s/\(\?:(.+?)\)\+/\1/g;
+ $other =~ s/\(\?:(.+?)\)\?/\1/g;
+ $other =~ s/\.\*\?//g;
+ push @MASK, $other;
+ # surround wildcards with capturing group for REGEX
+ s/(\.\*\?)/\(\1\)/g;
+ push @REGEX, $_;
+ }
+
+ $long_tokens = '\b' . join '\b|\b', grep length>=3,keys %{+{ map +($_=>undef), map /\w+/g, @REGEX }};
+ $long_tokens = $long_tokens . '\b';
+
+ parse_programme $target;
+}
+
+1; \ No newline at end of file

Return to:

Send suggestions and report system problems to the System administrator.