Update TokenStream.java

replacing with my implementation for TokenStream

Author: dayshanks

src/main/java/com/scanner/project/TokenStream.java

@@ -3,20 +3,13 @@
 
 // Implementation of the Scanner for KAY
 
-// This code DOES NOT implement a scanner for KAY yet. You have to complete
-// the code and also make sure it implements a scanner for KAY - not something
-// else, not more and not less
-
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 
 public class TokenStream {
 
-	// READ THE COMPLETE FILE FIRST
-	// You will need to adapt it to KAY, NOT JAY
-
 	// Instance variables 
 	private boolean isEof = false; // is end of file
 	private char nextChar = ' '; // next character in input stream
@@ -34,16 +27,14 @@ public TokenStream(String fileName) {
 			input = new BufferedReader(new FileReader(fileName));
 		} catch (FileNotFoundException e) {
 			System.out.println("File not found: " + fileName);
-			// System.exit(1); // Removed to allow ScannerDemo to continue
-			// running after the input file is not found.
 			isEof = true;
 		}
 	}
 
 	public Token nextToken() { // Main function of the scanner
-								// Return next token with its type and value.
+								// Return next token type and value.
 		Token t = new Token();
-		t.setType("Other"); // For now it is Other. You will update it in the code
+		t.setType("Other"); // For now it is Other
 		t.setValue("");
 
 		// First check for whitespaces and bypass them
@@ -52,12 +43,14 @@ public Token nextToken() { // Main function of the scanner
 		// Then check for a comment, and bypass it
 		// but remember that / may also be a division operator.
 		while (nextChar == '/') {
-			// The use of while (instead of if) prevents the 2nd line to be printed when
-			// there are two comment lines in a row.
 			nextChar = readChar();
 			if (nextChar == '/') { // If / is followed by another /
 				// skip rest of line - it's a comment.
-				// TODO TO BE COMPLETED
+				// look for <cr>, <lf>, <ff>
+				while (!isEof && !isEndOfLine(nextChar)) {
+					nextChar = readChar();
+				}
+				skipWhiteSpace();
 			} else {
 				// A slash followed by anything else must be an operator.
 				t.setValue("/");
@@ -72,17 +65,49 @@ public Token nextToken() { // Main function of the scanner
 			t.setType("Operator");
 			t.setValue(t.getValue() + nextChar);
 			switch (nextChar) {
-			// TODO TO BE COMPLETED OR CHANGED WHERE NEEDED TO IMPLEMENT KAY
 			case '<':
-				// <=
+				// <= or <>
+				nextChar = readChar();
+				if (nextChar == '=' || nextChar == '>') {
+					t.setValue(t.getValue() + nextChar);
+					nextChar = readChar();
+				}
+				return t;
 			case '>':
 				// >=
+				nextChar = readChar();
+				if (nextChar == '=') {
+					t.setValue(t.getValue() + nextChar);
+					nextChar = readChar();
+				}
+				return t;
 			case '=':
 				// ==
+				nextChar = readChar();
+				if (nextChar == '=') {
+					t.setValue(t.getValue() + nextChar);
+					nextChar = readChar();
+				}
+				return t;
 			case '!':
 				// !=
 				nextChar = readChar();
+				if (nextChar == '=') {
+					t.setValue(t.getValue() + nextChar);
+					nextChar = readChar();
+				}
 				return t;
+			case ':':
+				// := (assignment operator in KAY)
+				nextChar = readChar();
+				if (nextChar == '=') {
+					t.setValue(t.getValue() + nextChar);
+					nextChar = readChar();
+					return t;
+				} else {
+					t.setType("Other");
+					return t;
+				}
 			case '|':
 				// Look for ||
 				nextChar = readChar();
@@ -94,6 +119,7 @@ public Token nextToken() { // Main function of the scanner
 					t.setType("Other");
 				}
 				return t;
+
 			case '&':
 				// Look for &&
 				nextChar = readChar();
@@ -104,9 +130,9 @@ public Token nextToken() { // Main function of the scanner
 				} else {
 					t.setType("Other");
 				}
-
 				return t;
-			default: // all other operators
+
+			default: // all other operators: +, -, *, /
 				nextChar = readChar();
 				return t;
 			}
@@ -115,22 +141,24 @@ public Token nextToken() { // Main function of the scanner
 		// Then check for a separator
 		if (isSeparator(nextChar)) {
 			t.setType("Separator");
-			// TODO TO BE COMPLETED
+			t.setValue(t.getValue() + nextChar);
+			nextChar = readChar();
 			return t;
 		}
 
-		// Then check for an identifier, keyword, or literal (True or False).
+		// Then check for an identifier, keyword, or literal.
 		if (isLetter(nextChar)) {
 			// Set to an identifier
 			t.setType("Identifier");
 			while ((isLetter(nextChar) || isDigit(nextChar))) {
 				t.setValue(t.getValue() + nextChar);
 				nextChar = readChar();
 			}
-			// Now see if this is a keyword
+			// now see if this is a keyword
 			if (isKeyword(t.getValue())) {
 				t.setType("Keyword");
 			} else if (t.getValue().equals("True") || t.getValue().equals("False")) {
+				// In KAY, True and False are uppercased literals
 				t.setType("Literal");
 			}
 			if (isEndOfToken(nextChar)) { // If token is valid, returns.
@@ -187,30 +215,12 @@ private char readChar() {
 	}
 
 	private boolean isKeyword(String s) {
-		// TODO TO BE COMPLETED 
-		return false;
+		// Keywords in KAY: main, if, else, while, integer, bool
+		// Note: void is NOT a keyword in KAY
+		return s.equals("main") || s.equals("if") || s.equals("else") || 
+		       s.equals("while") || s.equals("integer") || s.equals("bool");
 	}
 
-	private boolean isSeparator(char c) {
-		// TODO TO BE COMPLETED
-		return false;
-	}
-
-	private boolean isOperator(char c) {
-		// Checks for characters that start operators
-		// TODO TO BE COMPLETED
-		return false;
-	}
-
-	private boolean isLetter(char c) {
-		return (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z');
-	}
-
-	private boolean isDigit(char c) {
-		// TODO TO BE COMPLETED
-		return false;
-	}
-	
 	private boolean isWhiteSpace(char c) {
 		return (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f');
 	}
@@ -219,7 +229,7 @@ private boolean isEndOfLine(char c) {
 		return (c == '\r' || c == '\n' || c == '\f');
 	}
 
-	private boolean isEndOfToken(char c) { // Is the value a seperate token?
+	private boolean isEndOfToken(char c) { // Is the value a separate token?
 		return (isWhiteSpace(nextChar) || isOperator(nextChar) || isSeparator(nextChar) || isEof);
 	}
 
@@ -230,7 +240,34 @@ private void skipWhiteSpace() {
 		}
 	}
 
+	private boolean isSeparator(char c) {
+		// Separators in KAY: { } ; ( )
+		return (c == '{' || c == '}' || c == ';' || c == '(' || c == ')');
+	}
+
+	private boolean isOperator(char c) {
+		// Checks for characters that start operators
+		// Operators in KAY: + - * / < <= > >= == != && || ! :=
+		return (c == '+' || c == '-' || c == '*' || c == '/' || 
+		        c == '<' || c == '>' || c == '=' || c == '!' || 
+		        c == '&' || c == '|' || c == ':');
+	}
+
+	private boolean isLetter(char c) {
+		return (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z');
+	}
+
+	private boolean isDigit(char c) {
+		return (c >= '0' && c <= '9');
+	}
+
+	public boolean isEndofFile() {
+		return isEof;
+	}
+}
+
 	public boolean isEndofFile() {
 		return isEof;
 	}
-}
+
+}