fish-shell/tokenizer.cpp

/** \file tokenizer.c

A specialized tokenizer for tokenizing the fish language. In the
future, the tokenizer should be extended to support marks,
tokenizing multiple strings and disposing of unused string
segments.
*/

#include "config.h"

#include <stdlib.h>
#include <stdio.h>
#include <wchar.h>
#include <wctype.h>
#include <string.h>
#include <unistd.h>


#include "fallback.h"
#include "util.h"

#include "wutil.h"
#include "tokenizer.h"
#include "common.h"

/* Wow what a hack */
#define TOK_CALL_ERROR(t, e, x) do { tok_call_error((t), (e), (t)->squash_errors ? L"" : (x)); } while (0)

/**
   Error string for unexpected end of string
*/
#define QUOTE_ERROR _( L"Unexpected end of string, quotes are not balanced" )

/**
   Error string for mismatched parenthesis
*/
#define PARAN_ERROR _( L"Unexpected end of string, parenthesis do not match" )

/**
   Error string for invalid redirections
*/
#define REDIRECT_ERROR _( L"Invalid input/output redirection" )

/**
   Error string for when trying to pipe from fd 0
*/
#define PIPE_ERROR _( L"Can not use fd 0 as pipe output" )

/**
   Characters that separate tokens. They are ordered by frequency of occurrence to increase parsing speed.
*/
#define SEP L" \n|\t;#\r<>^&"

/**
   Maximum length of a string containing a file descriptor number
*/
#define FD_STR_MAX_LEN 16

/**
   Descriptions of all tokenizer errors
*/
static const wchar_t *tok_desc[] =
{
  N_(L"Tokenizer not yet initialized"),
  N_( L"Tokenizer error" ),
  N_( L"Invalid token" ),
  N_( L"String" ),
  N_( L"Pipe" ),
  N_( L"End of command" ),
  N_( L"Redirect output to file" ),
  N_( L"Append output to file" ),
  N_( L"Redirect input to file" ),
  N_( L"Redirect to file descriptor" ),
  N_( L"Redirect output to file if file does not exist" ),
  N_( L"Run job in background" ),
  N_( L"Comment" )
}
  ;

/**
   Tests if the tokenizer buffer is large enough to hold contents of
   the specified length, and if not, reallocates the tokenizer buffer.

   \return 0 if the system could not provide the memory needed, and 1 otherwise.
*/
static int check_size( tokenizer *tok, size_t len )
{
  if( tok->last_len <= len )
  {
    wchar_t *tmp;
    tok->last_len = len +1;
    tmp = (wchar_t *)realloc( tok->last, sizeof(wchar_t)*tok->last_len );
    if( tmp == 0 )
    {
      wperror( L"realloc" );
      return 0;
    }
    tok->last = tmp;
  }
  return 1;
}

/**
   Set the latest tokens string to be the specified error message
*/
static void tok_call_error( tokenizer *tok, int error_type, const wchar_t *error_message )
{
  tok->last_type = TOK_ERROR;
  tok->error = error_type;
  if( !check_size( tok, wcslen( error_message)+1 ))
  {
    if( tok->last != 0 )
      *tok->last=0;
    return;
  }

  wcscpy( tok->last, error_message );
}

int tok_get_error( tokenizer *tok )
{
  return tok->error;
}


void tok_init( tokenizer *tok, const wchar_t *b, int flags )
{

    /* We can only generate error messages on the main thread due to wgettext() thread safety issues. */
    if (! (flags & TOK_SQUASH_ERRORS)) {
        ASSERT_IS_MAIN_THREAD();
    }

  CHECK( tok, );

  memset( tok, 0, sizeof( tokenizer) );

  CHECK( b, );


  tok->accept_unfinished = !! (flags & TOK_ACCEPT_UNFINISHED);
  tok->show_comments = !! (flags & TOK_SHOW_COMMENTS);
    tok->squash_errors = !! (flags & TOK_SQUASH_ERRORS);
  tok->has_next=true;

  tok->has_next = (*b != L'\0');
  tok->orig_buff = tok->buff = b;
    tok->cached_lineno_offset = 0;
    tok->cached_lineno_count = 0;
  tok_next( tok );
}

void tok_destroy( tokenizer *tok )
{
  CHECK( tok, );

  free( tok->last );
}

int tok_last_type( tokenizer *tok )
{
  CHECK( tok, TOK_ERROR );
  CHECK( tok->buff, TOK_ERROR );

  return tok->last_type;
}

wchar_t *tok_last( tokenizer *tok )
{
  CHECK( tok, 0 );

  return tok->last;
}

int tok_has_next( tokenizer *tok )
{
  /*
    Return 1 on broken tokenizer
  */
  CHECK( tok, 1 );
  CHECK( tok->buff, 1 );

/*  fwprintf( stderr, L"has_next is %ls \n", tok->has_next?L"true":L"false" );*/
  return   tok->has_next;
}

int tokenizer::line_number_of_character_at_offset(size_t offset)
{
    // we want to return (one plus) the number of newlines at offsets less than the given offset
    // cached_lineno_count is the number of newlines at indexes less than cached_lineno_offset
    const wchar_t *str = orig_buff;
  if (! str)
    return 0;

    // easy hack to handle 0
    if (offset == 0)
        return 1;

    size_t i;
    if (offset > cached_lineno_offset)
    {
        for ( i = cached_lineno_offset; str[i] && i<offset; i++)
        {
            /* Add one for every newline we find in the range [cached_lineno_offset, offset) */
            if( str[i] == L'\n' )
                cached_lineno_count++;
        }
        cached_lineno_offset = i; //note: i, not offset, in case offset is beyond the length of the string
    }
    else if (offset < cached_lineno_offset)
    {
        /* Subtract one for every newline we find in the range [offset, cached_lineno_offset) */
        for (i = offset; i < cached_lineno_offset; i++)
        {
            if (str[i] == L'\n')
                cached_lineno_count--;
        }
        cached_lineno_offset = offset;
    }
  return cached_lineno_count + 1;
}

/**
   Tests if this character can be a part of a string. The redirect ^ is allowed unless it's the first character.
*/
static bool is_string_char( wchar_t c, bool is_first )
{
    switch (c)
    {
        /* Unconditional separators */
        case L'\0':
        case L' ':
        case L'\n':
        case L'|':
        case L'\t':
        case L';':
        case L'#':
        case L'\r':
        case L'<':
        case L'>':
        case L'&':
            return false;

        /* Conditional separator */
        case L'^':
            return ! is_first;

        default:
            return true;

    }
}

/**
   Quick test to catch the most common 'non-magical' characters, makes
   read_string slightly faster by adding a fast path for the most
   common characters. This is obviously not a suitable replacement for
   iswalpha.
*/
static int myal( wchar_t c )
{
  return (c>=L'a' && c<=L'z') || (c>=L'A'&&c<=L'Z');
}

/**
   Read the next token as a string
*/
static void read_string( tokenizer *tok )
{
  const wchar_t *start;
  long len;
  int mode=0;
  int do_loop=1;
  int paran_count=0;

  start = tok->buff;
    bool is_first = true;

  while( 1 )
  {

    if( !myal( *tok->buff ) )
    {
//      debug(1, L"%lc", *tok->buff );

      if( *tok->buff == L'\\' )
      {
        tok->buff++;
        if( *tok->buff == L'\0' )
        {
          if( (!tok->accept_unfinished) )
          {
            TOK_CALL_ERROR( tok, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR );
            return;
          }
          else
          {
            do_loop = 0;
          }


        }
        else if( *tok->buff == L'\n' && mode == 0)
        {
          tok->buff--;
          do_loop = 0;
          break;
        }

        tok->buff++;
        continue;
      }


      /*
        The modes are as follows:

        0: regular text
        1: inside of subshell
        2: inside of array brackets
        3: inside of array brackets and subshell, like in '$foo[(ech'
      */
      switch( mode )
      {
        case 0:
        {
          switch( *tok->buff )
          {
            case L'(':
            {
              paran_count=1;
              mode = 1;
              break;
            }

            case L'[':
            {
              if( tok->buff != start )
                mode=2;
              break;
            }

            case L'\'':
            case L'"':
            {

              const wchar_t *end = quote_end( tok->buff );
              tok->last_quote = *tok->buff;
              if( end )
              {
                tok->buff=(wchar_t *)end;
              }
              else
              {
                tok->buff += wcslen( tok->buff );

                if( (!tok->accept_unfinished) )
                {
                  TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
                  return;
                }
                do_loop = 0;

              }
              break;
            }

            default:
            {
              if( !is_string_char(*(tok->buff), is_first) )
              {
                do_loop=0;
              }
            }
          }
          break;
        }

        case 3:
        case 1:
          switch( *tok->buff )
          {
            case L'\'':
            case L'\"':
            {
              const wchar_t *end = quote_end( tok->buff );
              if( end )
              {
                tok->buff=(wchar_t *)end;
              }
              else
              {
                tok->buff += wcslen( tok->buff );
                if( (!tok->accept_unfinished) )
                {
                  TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
                  return;
                }
                do_loop = 0;
              }

              break;
            }

            case L'(':
              paran_count++;
              break;
            case L')':
              paran_count--;
              if( paran_count == 0 )
              {
                mode--;
              }
              break;
            case L'\0':
              do_loop = 0;
              break;
          }
          break;
        case 2:
          switch( *tok->buff )
          {
            case L'(':
              paran_count=1;
              mode = 3;
              break;

            case L']':
              mode=0;
              break;

            case L'\0':
              do_loop = 0;
              break;
          }
          break;
      }
    }


    if( !do_loop )
      break;

    tok->buff++;
        is_first = false;
  }

  if( (!tok->accept_unfinished) && (mode!=0) )
  {
    TOK_CALL_ERROR( tok, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR );
    return;
  }


  len = tok->buff - start;

  if( !check_size( tok, len ))
    return;

  memcpy( tok->last, start, sizeof(wchar_t)*len );
  tok->last[len] = L'\0';
  tok->last_type = TOK_STRING;
}

/**
   Read the next token as a comment.
*/
static void read_comment( tokenizer *tok )
{
  const wchar_t *start;

  start = tok->buff;
  while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
    tok->buff++;

  size_t len = tok->buff - start;
  if( !check_size( tok, len ))
    return;

  memcpy( tok->last, start, sizeof(wchar_t)*len );
  tok->last[len] = L'\0';
  tok->last_type = TOK_COMMENT;
}

/**
   Read a FD redirection.
*/
static void read_redirect( tokenizer *tok, int fd )
{
  int mode = -1;

  if( (*tok->buff == L'>') ||
    (*tok->buff == L'^') )
  {
    tok->buff++;
    if( *tok->buff == *(tok->buff-1) )
    {
      tok->buff++;
      mode = 1;
    }
    else
    {
      mode = 0;
    }

    if( *tok->buff == L'|' )
    {
      if( fd == 0 )
      {
        TOK_CALL_ERROR( tok, TOK_OTHER, PIPE_ERROR );
        return;
      }
      check_size( tok, FD_STR_MAX_LEN );
      tok->buff++;
      swprintf( tok->last, FD_STR_MAX_LEN, L"%d", fd );
      tok->last_type = TOK_PIPE;
      return;
    }
  }
  else if( *tok->buff == L'<' )
  {
    tok->buff++;
    mode = 2;
  }
  else
  {
    TOK_CALL_ERROR( tok, TOK_OTHER, REDIRECT_ERROR);
  }

  if( !check_size( tok, 2 ))
  {
    return;
  }

  swprintf( tok->last, tok->last_len, L"%d", fd );

  if( *tok->buff == L'&' )
  {
    tok->buff++;
    tok->last_type = TOK_REDIRECT_FD;
  }
  else if( *tok->buff == L'?' )
  {
    tok->buff++;
    tok->last_type = TOK_REDIRECT_NOCLOB;
  }
  else
  {
    tok->last_type = TOK_REDIRECT_OUT + mode;
  }
}

wchar_t tok_last_quote( tokenizer *tok )
{
  CHECK( tok, 0 );

  return tok->last_quote;
}

/**
   Test if a character is whitespace. Differs from iswspace in that it
   does not consider a newline to be whitespace.
*/
static int my_iswspace( wchar_t c )
{
  if( c == L'\n' )
    return 0;
  else
    return iswspace( c );
}


const wchar_t *tok_get_desc( int type )
{
  if( type < 0 || (size_t)type >= sizeof( tok_desc ) )
  {
    return _(L"Invalid token type");
  }
  return _(tok_desc[type]);
}


void tok_next( tokenizer *tok )
{

  CHECK( tok, );
  CHECK( tok->buff, );

  if( tok_last_type( tok ) == TOK_ERROR )
  {
    tok->has_next=false;
    return;
  }

  if( !tok->has_next )
  {
/*    wprintf( L"EOL\n" );*/
    tok->last_type = TOK_END;
    return;
  }

  while( 1 )
  {
    if( my_iswspace(*(tok->buff) ) )
    {
      tok->buff++;
    }
    else
    {
      if(( *(tok->buff) == L'\\') &&( *(tok->buff+1) == L'\n') )
      {
        tok->last_pos = tok->buff - tok->orig_buff;
        tok->buff+=2;
        tok->last_type = TOK_END;
        return;
      }
      break;
    }
  }


  if( *tok->buff == L'#')
  {
    if( tok->show_comments )
    {
      tok->last_pos = tok->buff - tok->orig_buff;
      read_comment( tok );
      return;
    }
    else
    {
      while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
        tok->buff++;
    }

    while( my_iswspace(*(tok->buff) ) )
      tok->buff++;
  }

  tok->last_pos = tok->buff - tok->orig_buff;

  switch( *tok->buff )
  {

    case L'\0':
      tok->last_type = TOK_END;
      /*fwprintf( stderr, L"End of string\n" );*/
      tok->has_next = false;
      break;
    case 13:
    case L'\n':
    case L';':
      tok->last_type = TOK_END;
      tok->buff++;
      break;
    case L'&':
      tok->last_type = TOK_BACKGROUND;
      tok->buff++;
      break;

    case L'|':
      check_size( tok, 2 );

      tok->last[0]=L'1';
      tok->last[1]=L'\0';
      tok->last_type = TOK_PIPE;
      tok->buff++;
      break;

    case L'>':
      read_redirect( tok, 1 );
      return;
    case L'<':
      read_redirect( tok, 0 );
      return;
    case L'^':
      read_redirect( tok, 2 );
      return;

    default:
    {
      if( iswdigit( *tok->buff ) )
      {
        const wchar_t *orig = tok->buff;
        int fd = 0;
        while( iswdigit( *tok->buff ) )
          fd = (fd*10) + (*(tok->buff++) - L'0');

        switch( *(tok->buff))
        {
          case L'^':
          case L'>':
          case L'<':
            read_redirect( tok, fd );
            return;
        }
        tok->buff = orig;
      }
      read_string( tok );
    }

  }

}

const wchar_t *tok_string( tokenizer *tok )
{
  return tok?tok->orig_buff:0;
}

wchar_t *tok_first( const wchar_t *str )
{
  tokenizer t;
  wchar_t *res=0;

  CHECK( str, 0 );

  tok_init( &t, str, TOK_SQUASH_ERRORS);

  switch( tok_last_type( &t ) )
  {
    case TOK_STRING:
//        fwprintf( stderr, L"Got token %ls\n", tok_last( &t ));
      res = wcsdup(tok_last( &t ));
      break;
    default:
      break;
  }

  tok_destroy( &t );
  return res;
}

int tok_get_pos( tokenizer *tok )
{
  CHECK( tok, 0 );

  return (int)tok->last_pos;
}


void tok_set_pos( tokenizer *tok, int pos )
{
  CHECK( tok, );

  tok->buff = tok->orig_buff + pos;
  tok->has_next = true;
  tok_next( tok );
}


#ifdef TOKENIZER_TEST

/**
   This main function is used for compiling the tokenizer_test command, used for testing the tokenizer.
*/
int main( int argc, char **argv )
{
  tokenizer tok;
  int i;
  for ( i=1; i<argc; i++ )
  {
    wprintf( L"Tokenizing string %s\n", argv[i] );
    for( tok_init( &tok, str2wcs(argv[i]), 0 ); tok_has_next( &tok ); tok_next( &tok ) )
    {
      switch( tok_last_type( &tok ) )
       {
        case TOK_INVALID:
          wprintf( L"Type: INVALID\n" );
          break;
        case TOK_STRING:
          wprintf( L"Type: STRING\t Value: %ls\n", tok_last( &tok ) );
          break;
        case TOK_PIPE:
          wprintf( L"Type: PIPE\n" );
          break;
        case TOK_END:
          wprintf( L"Type: END\n" );
          break;
        case TOK_ERROR:
          wprintf( L"Type: ERROR\n" );
          break;
        default:
          wprintf( L"Type: Unknown\n"  );
          break;
      }
    }
    tok_destroy( &tok );

  }
}

#endif