Changeset 2d232ed

Timestamp:

Mar 2, 2010, 9:14:44 PM (15 years ago)

Author:

David Benjamin <davidben@mit.edu>

Branches:

master, release-1.10, release-1.6, release-1.7, release-1.8, release-1.9

Children:

30c5aab

Parents:

af21934

git-author:

David Benjamin <davidben@mit.edu> (02/27/10 22:08:11)

git-committer:

David Benjamin <davidben@mit.edu> (03/02/10 21:14:44)

Message:

Move Completion::Context::tokenize into own module

Also add TODO for future work.

Signed-off-by: David Benjamin <davidben@mit.edu>

Location:

perl/lib/BarnOwl

Files:

• Completion/Context.pm (modified) (2 diffs)
• Parse.pm (added)

perl/lib/BarnOwl/Completion/Context.pm

@@ -41 +41 @@
 use base qw(Class::Accessor::Fast);
 use Carp qw(croak);
+use BarnOwl::Parse qw(tokenize);
 
 __PACKAGE__->mk_ro_accessors(qw(line point words word word_point
@@ -91 +92 @@
 }
 
-=for doc
-
-Ideally, this should use the same codepath we use to /actually/
-tokenize commands, but for now, make sure this is kept in sync with
-owl_parseline in util.c
-
-Unlike owl_parseline, we always return a result, even in the presence
-of parse errors, since we may be called on incomplete command-lines.
-
-The owl_parseline rules are:
-
-* Tokenize on ' ' and '\t'
-* ' and " are quote characters
-* \ has no effect
-
-=cut
-
-my $boring = qr{[^'" \t]};
-my $quote  = qr{['"]};
-my $space  = qr{[ \t]};
-
-sub tokenize {
-    my $line = shift;
-    my $point = shift;
-
-    my @words = ();
-    my $cword = 0;
-    my $cword_start;
-    my $cword_end;
-    my $word_point;
-
-    my $word = '';
-    my $wstart = 0;
-    my $skipped = 0;
-    my $have_word = 0;
-
-    pos($line) = 0;
-    while(pos($line) < length($line)) {
-        if($line =~ m{\G ($boring+) }gcx) {
-            $word .= $1;
-            $have_word = 1;
-        } elsif ($line =~ m{\G ($quote)}gcx) {
-            my $chr = $1;
-            $skipped++ if pos($line) > $point;
-            if($line =~ m{\G ([^$chr]*) $chr}gcx) {
-                $word .= $1;
-                $skipped++ if pos($line) > $point;
-            } else {
-                $word .= substr($line, pos($line));
-                pos($line) = length($line);
-            }
-            $have_word = 1;
-        }
-
-        if ($line =~ m{\G ($space+|$)}gcx) {
-            my $wend = pos($line) - length($1);
-            if ($have_word) {
-                push @words, $word;
-                $cword++ unless $wend >= $point;
-                if(($wend >= $point) && !defined($word_point)) {
-                    $word_point = length($word) - ($wend - $point) + $skipped;
-                    $cword_start = $wstart;
-                    $cword_end   = $wend;
-                }
-            }
-            # Always reset, so we get $wstart right
-            $word = '';
-            $wstart = pos($line);
-            $skipped = 0;
-            $have_word = 0;
-        }
-    }
-
-    if(length($word)) { die("Internal error, leftover=$word"); }
-
-    unless(defined($word_point)) {
-        $word_point = 0;
-        $cword_start = $cword_end = $point;
-    }
-
-    return (\@words, $cword, $word_point, $cword_start, $cword_end);
-}
-
 1;