Ignore:
Timestamp:
Mar 2, 2010, 9:14:44 PM (15 years ago)
Author:
David Benjamin <davidben@mit.edu>
Branches:
master, release-1.10, release-1.6, release-1.7, release-1.8, release-1.9
Children:
2d232ed
Parents:
ccafe64
git-author:
David Benjamin <davidben@mit.edu> (02/27/10 21:52:26)
git-committer:
David Benjamin <davidben@mit.edu> (03/02/10 21:14:44)
Message:
Fix leading whitespace in tokenizer and refactor

Should make tests pass. Leading whitespace now works. Also, we remove
the dummy '' append in tokenize() and move it as a post-processing step
in Context::new. This is so tokenize() is a little more generic.

Signed-off-by: David Benjamin <davidben@mit.edu>
File:
1 edited

Legend:

Unmodified
Added
Removed
  • perl/lib/BarnOwl/Completion/Context.pm

    re97c5d05 raf21934  
    5454    my ($words, $word, $word_point,
    5555        $word_start, $word_end) = tokenize($line, $point);
     56    push @$words, '' if scalar @$words <= $word;
    5657
    5758    my $self = {
     
    124125    my $wstart = 0;
    125126    my $skipped = 0;
     127    my $have_word = 0;
    126128
    127129    pos($line) = 0;
     
    129131        if($line =~ m{\G ($boring+) }gcx) {
    130132            $word .= $1;
     133            $have_word = 1;
    131134        } elsif ($line =~ m{\G ($quote)}gcx) {
    132135            my $chr = $1;
     
    139142                pos($line) = length($line);
    140143            }
     144            $have_word = 1;
    141145        }
    142146
    143147        if ($line =~ m{\G ($space+|$)}gcx) {
    144148            my $wend = pos($line) - length($1);
    145             push @words, $word;
    146             $cword++ unless $wend >= $point;
    147             if(($wend >= $point) && !defined($word_point)) {
    148                 $word_point = length($word) - ($wend - $point) + $skipped;
    149                 $cword_start = $wstart;
    150                 $cword_end   = $wend;
     149            if ($have_word) {
     150                push @words, $word;
     151                $cword++ unless $wend >= $point;
     152                if(($wend >= $point) && !defined($word_point)) {
     153                    $word_point = length($word) - ($wend - $point) + $skipped;
     154                    $cword_start = $wstart;
     155                    $cword_end   = $wend;
     156                }
    151157            }
     158            # Always reset, so we get $wstart right
    152159            $word = '';
    153160            $wstart = pos($line);
    154161            $skipped = 0;
     162            $have_word = 0;
    155163        }
    156164    }
     
    161169        $word_point = 0;
    162170        $cword_start = $cword_end = $point;
    163         push @words, '' if $point > 0;
    164171    }
    165172
Note: See TracChangeset for help on using the changeset viewer.