Simplify lexer_parse_number (parse_number) and add unicode support to the function.

JerryScript-DCO-1.0-Signed-off-by: Ruben Ayrapetyan [email protected]

Author: ruben-ayrapetyan

jerry-core/parser/js/lexer.cpp

@@ -794,34 +794,35 @@ lexer_parse_identifier_or_keyword (void)
   return ret;
 } /* lexer_parse_identifier_or_keyword */
 
-/* In this function we cannot use strtol function
-   since there is no octal literals in ECMAscript.  */
+/**
+ * Parse numeric literal (ECMA-262, v5, 7.8.3)
+ *
+ * @return token of TOK_SMALL_INT or TOK_NUMBER types
+ */
 static token
-parse_number (void)
+lexer_parse_number (void)
 {
   ecma_char_t c = LA (0);
   bool is_hex = false;
   bool is_fp = false;
-  bool is_exp = false;
-  bool is_overflow = false;
   ecma_number_t fp_res = .0;
   size_t tok_length = 0, i;
-  uint32_t res = 0;
   token known_token;
 
-  JERRY_ASSERT (isdigit (c) || c == '.');
+  JERRY_ASSERT (lit_char_is_decimal_digit (c)
+                || c == LIT_CHAR_DOT);
 
-  if (c == '0')
+  if (c == LIT_CHAR_0)
   {
-    if (LA (1) == 'x' || LA (1) == 'X')
+    if (LA (1) == LIT_CHAR_LOWERCASE_X
+        || LA (1) == LIT_CHAR_UPPERCASE_X)
     {
       is_hex = true;
     }
   }
-
-  if (c == '.')
+  else if (c == LIT_CHAR_DOT)
   {
-    JERRY_ASSERT (!isalpha (LA (1)));
+    JERRY_ASSERT (lit_char_is_decimal_digit (LA (1)));
     is_fp = true;
   }
 
@@ -834,189 +835,149 @@ parse_number (void)
     while (true)
     {
       c = LA (0);
-      if (!isxdigit (c))
+      if (!lit_char_is_hex_digit (c))
       {
         break;
       }
       consume_char ();
     }
 
-    if (isalpha (c) || c == '_' || c == '$')
+    if (lexer_is_char_can_be_identifier_start (c))
     {
-      PARSE_ERROR ("Integer literal shall not contain non-digit characters",
+      PARSE_ERROR ("Identifier just after integer literal",
                    lit_utf8_iterator_get_offset (&src_iter));
     }
 
     tok_length = (size_t) (lit_utf8_iterator_get_ptr (&src_iter) - token_start);
 
+    /* token is constructed at end of function */
     for (i = 0; i < tok_length; i++)
     {
-      if (!is_overflow)
-      {
-        res = (res << 4) + lit_char_hex_to_int (token_start[i]);
-      }
-      else
-      {
-        fp_res = fp_res * 16 + (ecma_number_t) lit_char_hex_to_int (token_start[i]);
-      }
-
-      if (res > 255)
-      {
-        fp_res = (ecma_number_t) res;
-        is_overflow = true;
-        res = 0;
-      }
-    }
-
-    if (is_overflow)
-    {
-      known_token = convert_seen_num_to_token (fp_res);
-      token_start = NULL;
-      return known_token;
-    }
-    else
-    {
-      known_token = create_token (TOK_SMALL_INT, (uint8_t) res);
-      token_start = NULL;
-      return known_token;
+      fp_res = fp_res * 16 + (ecma_number_t) lit_char_hex_to_int (token_start[i]);
     }
   }
-
-  JERRY_ASSERT (!is_hex && !is_exp);
-
-  new_token ();
-
-  // Eat up '.'
-  if (is_fp)
+  else
   {
-    consume_char ();
-  }
+    bool is_exp = false;
 
-  while (true)
-  {
-    c = LA (0);
-    if (is_fp && c == '.')
-    {
-      FIXME (/* This is wrong: 1..toString ().  */)
-      PARSE_ERROR ("Integer literal shall not contain more than one dot character",
-                   lit_utf8_iterator_get_offset (&src_iter));
-    }
-    if (is_exp && (c == 'e' || c == 'E'))
-    {
-      PARSE_ERROR ("Integer literal shall not contain more than exponential marker ('e' or 'E')",
-                   lit_utf8_iterator_get_offset (&src_iter));
-    }
+    new_token ();
 
-    if (c == '.')
+    // Eat up '.'
+    if (is_fp)
     {
-      if (isalpha (LA (1)) || LA (1) == '_' || LA (1) == '$')
-      {
-        PARSE_ERROR ("Integer literal shall not contain non-digit character after got character",
-                     lit_utf8_iterator_get_offset (&src_iter));
-      }
-      is_fp = true;
       consume_char ();
-      continue;
     }
 
-    if (c == 'e' || c == 'E')
+    while (true)
     {
-      if (LA (1) == '-' || LA (1) == '+')
+      c = LA (0);
+
+      if (c == LIT_CHAR_DOT)
       {
-        consume_char ();
+        if (is_fp)
+        {
+          /* token is constructed at end of function */
+          break;
+        }
+        else
+        {
+          is_fp = true;
+          consume_char ();
+
+          continue;
+        }
+      }
+      else if (c == LIT_CHAR_LOWERCASE_E
+               || c == LIT_CHAR_UPPERCASE_E)
+      {
+        if (is_exp)
+        {
+          PARSE_ERROR ("Numeric literal shall not contain more than exponential marker ('e' or 'E')",
+                       lit_utf8_iterator_get_offset (&src_iter));
+        }
+        else
+        {
+          is_exp = true;
+          consume_char ();
+
+          if (LA (0) == LIT_CHAR_MINUS
+              || LA (0) == LIT_CHAR_PLUS)
+          {
+            consume_char ();
+          }
+
+          continue;
+        }
       }
-      if (!isdigit (LA (1)))
+      else if (!lit_char_is_decimal_digit (c))
       {
-        PARSE_ERROR ("Integer literal shall not contain non-digit character after exponential marker ('e' or 'E')",
-                     lit_utf8_iterator_get_offset (&src_iter));
+        if (lexer_is_char_can_be_identifier_start (c))
+        {
+          PARSE_ERROR ("Numeric literal shall not contain non-numeric characters",
+                       lit_utf8_iterator_get_offset (&src_iter));
+        }
+
+        /* token is constructed at end of function */
+        break;
       }
-      is_exp = true;
+
       consume_char ();
-      continue;
     }
 
-    if (isalpha (c) || c == '_' || c == '$')
-    {
-      PARSE_ERROR ("Integer literal shall not contain non-digit characters",
-                   lit_utf8_iterator_get_offset (&src_iter));
-    }
+    tok_length = (size_t) (lit_utf8_iterator_get_ptr (&src_iter) - token_start);
 
-    if (!isdigit (c))
+    if (is_fp || is_exp)
     {
-      break;
-    }
-
-    consume_char ();
-  }
+      ecma_number_t res = ecma_utf8_string_to_number (token_start, (jerry_api_size_t) tok_length);
+      JERRY_ASSERT (!ecma_number_is_nan (res));
 
-  tok_length = (size_t) (lit_utf8_iterator_get_ptr (&src_iter) - token_start);
-  if (is_fp || is_exp)
-  {
-    ecma_number_t res = ecma_utf8_string_to_number (token_start, (jerry_api_size_t) tok_length);
-    JERRY_ASSERT (!ecma_number_is_nan (res));
-    known_token = convert_seen_num_to_token (res);
-    token_start = NULL;
-    return known_token;
-  }
+      known_token = convert_seen_num_to_token (res);
+      token_start = NULL;
 
-  if (*token_start == '0' && tok_length != 1)
-  {
-    if (strict_mode)
-    {
-      PARSE_ERROR ("Octal tnteger literals are not allowed in strict mode", token_start - buffer_start);
+      return known_token;
     }
-    for (i = 0; i < tok_length; i++)
+    else if (*token_start == LIT_CHAR_0
+             && tok_length != 1)
     {
-      if (!is_overflow)
+      /* Octal integer literals */
+      if (strict_mode)
       {
-        res = res * 8 + lit_char_hex_to_int (token_start[i]);
+        PARSE_ERROR ("Octal integer literals are not allowed in strict mode", token_start - buffer_start);
       }
       else
       {
-        fp_res = fp_res * 8 + (ecma_number_t) lit_char_hex_to_int (token_start[i]);
-      }
-      if (res > 255)
-      {
-        fp_res = (ecma_number_t) res;
-        is_overflow = true;
-        res = 0;
+        /* token is constructed at end of function */
+
+        for (i = 0; i < tok_length; i++)
+        {
+          fp_res = fp_res * 8 + (ecma_number_t) lit_char_hex_to_int (token_start[i]);
+        }
       }
     }
-  }
-  else
-  {
-    for (i = 0; i < tok_length; i++)
+    else
     {
-      if (!is_overflow)
-      {
-        res = res * 10 + lit_char_hex_to_int (token_start[i]);
-      }
-      else
+      /* token is constructed at end of function */
+
+      for (i = 0; i < tok_length; i++)
       {
         fp_res = fp_res * 10 + (ecma_number_t) lit_char_hex_to_int (token_start[i]);
       }
-      if (res > 255)
-      {
-        fp_res = (ecma_number_t) res;
-        is_overflow = true;
-        res = 0;
-      }
     }
   }
 
-  if (is_overflow)
+  if (fp_res >= 0 && fp_res <= 255 && (uint8_t) fp_res == fp_res)
   {
-    known_token = convert_seen_num_to_token (fp_res);
+    known_token = create_token (TOK_SMALL_INT, (uint8_t) fp_res);
     token_start = NULL;
     return known_token;
   }
   else
   {
-    known_token = create_token (TOK_SMALL_INT, (uint8_t) res);
+    known_token = convert_seen_num_to_token (fp_res);
     token_start = NULL;
     return known_token;
   }
-}
+} /* lexer_parse_number */
 
 /**
  * Parse string literal (ECMA-262 v5, 7.8.4)
@@ -1243,9 +1204,12 @@ lexer_next_token_private (void)
     return lexer_parse_identifier_or_keyword ();
   }
 
-  if (isdigit (c) || (c == '.' && isdigit (LA (1))))
+  /* ECMA-262 v5, 7.8.3, Numeric literal */
+  if (lit_char_is_decimal_digit (c)
+      || (c == LIT_CHAR_DOT
+          && lit_char_is_decimal_digit (LA (1))))
   {
-    return parse_number ();
+    return lexer_parse_number ();
   }
 
   if (c == '\n')