Changeset 2d232ed for perl


Ignore:
Timestamp:
Mar 2, 2010, 9:14:44 PM (14 years ago)
Author:
David Benjamin <davidben@mit.edu>
Branches:
master, release-1.10, release-1.6, release-1.7, release-1.8, release-1.9
Children:
30c5aab
Parents:
af21934
git-author:
David Benjamin <davidben@mit.edu> (02/27/10 22:08:11)
git-committer:
David Benjamin <davidben@mit.edu> (03/02/10 21:14:44)
Message:
Move Completion::Context::tokenize into own module

Also add TODO for future work.

Signed-off-by: David Benjamin <davidben@mit.edu>
Location:
perl/lib/BarnOwl
Files:
1 added
1 edited

Legend:

Unmodified
Added
Removed
  • perl/lib/BarnOwl/Completion/Context.pm

    raf21934 r2d232ed  
    4141use base qw(Class::Accessor::Fast);
    4242use Carp qw(croak);
     43use BarnOwl::Parse qw(tokenize);
    4344
    4445__PACKAGE__->mk_ro_accessors(qw(line point words word word_point
     
    9192}
    9293
    93 =for doc
    94 
    95 Ideally, this should use the same codepath we use to /actually/
    96 tokenize commands, but for now, make sure this is kept in sync with
    97 owl_parseline in util.c
    98 
    99 Unlike owl_parseline, we always return a result, even in the presence
    100 of parse errors, since we may be called on incomplete command-lines.
    101 
    102 The owl_parseline rules are:
    103 
    104 * Tokenize on ' ' and '\t'
    105 * ' and " are quote characters
    106 * \ has no effect
    107 
    108 =cut
    109 
    110 my $boring = qr{[^'" \t]};
    111 my $quote  = qr{['"]};
    112 my $space  = qr{[ \t]};
    113 
    114 sub tokenize {
    115     my $line = shift;
    116     my $point = shift;
    117 
    118     my @words = ();
    119     my $cword = 0;
    120     my $cword_start;
    121     my $cword_end;
    122     my $word_point;
    123 
    124     my $word = '';
    125     my $wstart = 0;
    126     my $skipped = 0;
    127     my $have_word = 0;
    128 
    129     pos($line) = 0;
    130     while(pos($line) < length($line)) {
    131         if($line =~ m{\G ($boring+) }gcx) {
    132             $word .= $1;
    133             $have_word = 1;
    134         } elsif ($line =~ m{\G ($quote)}gcx) {
    135             my $chr = $1;
    136             $skipped++ if pos($line) > $point;
    137             if($line =~ m{\G ([^$chr]*) $chr}gcx) {
    138                 $word .= $1;
    139                 $skipped++ if pos($line) > $point;
    140             } else {
    141                 $word .= substr($line, pos($line));
    142                 pos($line) = length($line);
    143             }
    144             $have_word = 1;
    145         }
    146 
    147         if ($line =~ m{\G ($space+|$)}gcx) {
    148             my $wend = pos($line) - length($1);
    149             if ($have_word) {
    150                 push @words, $word;
    151                 $cword++ unless $wend >= $point;
    152                 if(($wend >= $point) && !defined($word_point)) {
    153                     $word_point = length($word) - ($wend - $point) + $skipped;
    154                     $cword_start = $wstart;
    155                     $cword_end   = $wend;
    156                 }
    157             }
    158             # Always reset, so we get $wstart right
    159             $word = '';
    160             $wstart = pos($line);
    161             $skipped = 0;
    162             $have_word = 0;
    163         }
    164     }
    165 
    166     if(length($word)) { die("Internal error, leftover=$word"); }
    167 
    168     unless(defined($word_point)) {
    169         $word_point = 0;
    170         $cword_start = $cword_end = $point;
    171     }
    172 
    173     return (\@words, $cword, $word_point, $cword_start, $cword_end);
    174 }
    175 
    176941;
Note: See TracChangeset for help on using the changeset viewer.