import java.io.FileReader;
import java.io.PushbackReader;
import java.io.IOException;

/**
 * A simple tokenizer for CS 212, Feb 2004.
 * 
 * @author Paul Chew
 */
public class SimpleTokenizer {
  
  PushbackReader in;             // Input file
  boolean pushedBack = false;    // True iff token has been pushed back
  String mostRecent = null;      // Most recent token returned (via next())
  
  /**
   * Create a simple tokenizer reading tokens from fileName.
   */
  public SimpleTokenizer (String fileName) throws IOException {
    in = new PushbackReader(new FileReader(fileName));
  }
  
  /**
   * Close the input file.
   */
  public void close () throws IOException {
    in.close();
  }
  
  /**
   * Push a token back to the tokenizer.  Causes the current token to appear
   * again when next() is used.
   */
  public void pushBack () {
    pushedBack = true;
  }
  
  /**
   * Next token from input.
   */
  public String next () throws IOException {
    char ch;                   // An input character
    String token;              // The new token
    if (pushedBack) {
      pushedBack = false;
      return mostRecent;
    }
    ch = (char) in.read();
    while (Character.isWhitespace(ch)) ch = (char) in.read();
    if (Character.isLetter(ch)) {
      token = "";
      while (Character.isLetterOrDigit(ch)) {
        token = token + ch;
        ch = (char) in.read();
      }
      in.unread(ch);
    } else if (Character.isDigit(ch)) {
      token = "";
      while (Character.isDigit(ch)) {
        token = token + ch;
        ch = (char) in.read();
      }
      in.unread(ch);
    } else if (",;=+-*/:.".indexOf(ch) >= 0) {
      token = Character.toString(ch);
    } else throw new RuntimeException("Bad character: " + ch);
    mostRecent = token;
    return token;
  }
  
  /**
   * Some test code.
   */
  public static void main (String[] args) throws IOException {
    SimpleTokenizer tokenizer = new SimpleTokenizer("test.txt");
    while (true) System.out.println("token: " + tokenizer.next());
  }
}