[kaffe] Bug report (java.io.StreamTokenizer)

[Ito Kazumitsu]

>>>>> "*" == Ito Kazumitsu <kaz@maczuka.gcd.org> writes:

*> Further guessing on how Sun's StreamTokenizer is working is needed.

Afer a lot of try-and-error testing,  I found that the following
patch gives the same result as Sun's.  I will commit it after
making a document about this change.

--- java/io/StreamTokenizer.java.orig	Tue Feb 19 09:48:11 2002
+++ java/io/StreamTokenizer.java	Wed Jul  2 00:41:47 2003
@@ -74,7 +74,12 @@
 
 public void commentChar(int ch) {
 	if (ch >= 0 && ch <= 255) {
-		lookup(ch).isComment = true;
+		TableEntry e = lookup(ch);
+		e.isComment = true;
+		e.isStringQuote = false;
+		e.isWhitespace = false;
+		e.isAlphabetic = false;
+		e.isNumeric = false;
 	}
 }
 
@@ -116,14 +121,6 @@
 		/* Skip whitespace and return nextTokenType */
 		parseWhitespaceChars(chr);
 	}
-	else if (e.isNumeric) {
-		/* Parse the number and return */
-		parseNumericChars(chr);
-	}
-	else if (e.isAlphabetic) {
-		/* Parse the word and return */
-		parseAlphabeticChars(chr);
-	}
 	/* Contrary to the description in JLS 1.ed,
 	   C & C++ comments seem to be checked
 	   before other comments. That actually
@@ -142,6 +139,14 @@
 	        /* skip comment and return nextTokenType() */
 	        parseCommentChars();
 	}
+	else if (e.isNumeric) {
+		/* Parse the number and return */
+		parseNumericChars(chr);
+	}
+	else if (e.isAlphabetic) {
+		/* Parse the word and return */
+		parseAlphabeticChars(chr);
+	}
 	else if (e.isStringQuote) {
 	        /* Parse string and return word */
 	        parseStringQuoteChars(chr);
@@ -231,7 +236,7 @@
 		buffer.append((char)chr);
 		chr = chrRead();
 
-	} while (lookup(chr).isNumeric
+	} while (lookup(chr).isAnywayNumeric
 		 && chr != '-'
 		 && !(chr == '.' && dotParsed));
 
@@ -411,6 +416,7 @@
 		e.isAlphabetic = false;
 		e.isStringQuote = false;
 		e.isNumeric = false;
+		e.isAnywayNumeric = false;
 		e.isComment = false;
 		e.isWhitespace = false;
 	}
@@ -432,10 +438,20 @@
 
 public void parseNumbers() {
 	for (int letter = '0'; letter <= '9'; letter++) {
-		lookup(letter).isNumeric = true;
+		setNumeric(letter);
 	}
-	lookup('.').isNumeric = true;
-	lookup('-').isNumeric = true;
+	setNumeric('.');
+	setNumeric('-');
+}
+
+private void setNumeric(int letter) {
+	TableEntry e = lookup(letter);
+	e.isNumeric = true;
+	e.isAnywayNumeric = true;
+	// e.isWhitespace = false;
+	e.isStringQuote = false;
+	e.isComment = false;
+	e.isAlphabetic = false;
 }
 
 public void pushBack() {
@@ -444,7 +460,12 @@
 
 public void quoteChar(int ch) {
 	if (ch >= 0 && ch <= 255) {
-		lookup(ch).isStringQuote = true;
+		TableEntry e = lookup(ch);
+		e.isComment = false;
+		e.isStringQuote = true;
+		e.isWhitespace = false;
+		e.isAlphabetic = false;
+		e.isNumeric = false;
 	}
 }
 
@@ -547,6 +568,8 @@
 	for (int letter = low; letter <= hi; letter++) {
 		TableEntry e = lookup(letter);
 		e.isWhitespace = true;
+		e.isComment = false;
+		e.isStringQuote = false;
 		e.isAlphabetic = false;
 		e.isNumeric = false;
 	}    
@@ -562,7 +585,12 @@
 	}
 
 	for (int letter = low; letter <= hi; letter++) {
-		lookup(letter).isAlphabetic = true;
+		TableEntry e = lookup(letter);
+		e.isAlphabetic = true;
+		e.isComment = false;
+		e.isStringQuote = false;
+		// e.isWhitespace = false;
+		// e.isNumeric = false;
 	}    
 }
 
@@ -576,6 +604,7 @@
 class TableEntry {
 
 private boolean isNumeric;
+private boolean isAnywayNumeric;
 private boolean isWhitespace;
 private boolean isAlphabetic;
 private boolean isStringQuote;