001 /* Copyright 2000, 2001, Compaq Computer Corporation */
002
003 package javafe.parser;
004
005 import javafe.ast.Identifier;
006 import javafe.ast.LexicalPragma;
007 import javafe.ast.ModifierPragma;
008 import javafe.ast.TypeModifierPragma;
009 import javafe.ast.StmtPragma;
010 import javafe.ast.TypeDeclElemPragma;
011 import javafe.ast.PrettyPrint;
012
013 import javafe.util.Assert;
014 import javafe.util.Location;
015
016 /**
017 * The <code>Token</code> class defines a set of fields that describe
018 * lexical tokens.
019 *
020 * <p> Allocating an individual <code>Token</code> object for every
021 * token consumes a lot of memory. Instead, our front end tries to
022 * reuse <code>Token</code> instances as much as possible. Thus, for
023 * example, <code>Lex</code> is a subclass of <code>Token</code>;
024 * <code>Lex</code> returns information about the current token not by
025 * allocating a new <code>Token</code> object but rather by filling in
026 * the <code>Token</code> fields of itself. To facilitate the reuse
027 * of <code>Token</code> instances, <code>Token</code> has a
028 * <code>copyInto</code> method.
029 */
030
031 public class Token
032 {
033 /***************************************************
034 * *
035 * Instance fields: *
036 * *
037 **************************************************/
038
039 /**
040 * Integer code giving the kind of token.
041 *
042 * <p> To clear a token, set this field to Token.CLEAR and set
043 * identifierVal and auxVal to null, as is done in the
044 * clear() method. </p>
045 *
046 * @see javafe.parser.TagConstants
047 */
048 public int ttype = CLEAR;
049
050 //* The token code to use to clear a token; EOF for now.
051 public static final int CLEAR = TagConstants.EOF;
052
053 /** Clear the current token. */
054 //@ ensures ttype == CLEAR;
055 //@ ensures identifierVal == null;
056 //@ ensures auxVal == null;
057 public void clear() {
058 ttype = CLEAR;
059 identifierVal = null;
060 auxVal = null;
061 }
062
063 /** The location of the first character of the token. */
064 //@ invariant ttype != CLEAR ==> startingLoc != Location.NULL;
065 public int startingLoc;
066
067 /**
068 * The location of the last character of the token. (This value
069 * isn't "off-by-one" right now.)
070 */
071 //@ invariant ttype != CLEAR ==> endingLoc != Location.NULL;
072 public int endingLoc;
073
074
075 /**
076 * Identifier represented by the token. Must be non-null if
077 * <code>ttype</code> is <TT>TagConstants.IDENT</TT>.
078 */
079 /*@ invariant (ttype==TagConstants.IDENT) ==> (identifierVal != null); */
080 public Identifier identifierVal;
081
082 /**
083 * Auxillary information about the token. In the case of literal
084 * tokens, this field holds the value of token. In particular,
085 * if <code>ttype</code> is one of the codes on the left of the
086 * following table, then <code>auxVal</code> must be an instance
087 * of the class on the right:
088 *
089 * <center><code><table>
090 * <tr><td> TagConstants.INTLIT </td> <td> Integer </td></tr>
091 * <tr><td> TagConstants.LONGLIT </td> <td> Long </td></tr>
092 * <tr><td> TagConstants.FLOATLIT </td> <td> Float </td></tr>
093 * <tr><td> TagConstants.DOUBLELIT </td> <td> Double </td></tr>
094 * <tr><td> TagConstants.STRINGLIT </td> <td> String </td></tr>
095 * <tr><td> TagConstants.CHARLIT </td> <td> Integer </td></tr>
096 * <tr><td> TagConstants.LEXICALPRAGMA </td>
097 * <td> LexicalPragma </td></tr>
098 * <tr><td> TagConstants.MODIFIERPRAGMA </td>
099 * <td> ModifierPragma </td></tr>
100 * <tr><td> TagConstants.STMTPRAGMA </td> <td> StmtPragma </td></tr>
101 * <tr><td> TagConstants.TYPEDECLELEMPRAGMA </td>
102 * <td> TypeDeclElemPragma</td></tr>
103 * </table> </code> </center><p>
104 *
105 * For the various pragmas, <code>auxVal</code> may be
106 * <code>null</code>, but for the literals it may <em>not</em>
107 * be.
108 */
109 /*@ invariant (
110 !(ttype==TagConstants.BOOLEANLIT) &&
111 (ttype==TagConstants.INTLIT ==> auxVal instanceof Integer) &&
112 (ttype==TagConstants.LONGLIT ==> auxVal instanceof Long) &&
113 (ttype==TagConstants.FLOATLIT ==> auxVal instanceof Float) &&
114 (ttype==TagConstants.DOUBLELIT ==> auxVal instanceof Double) &&
115 (ttype==TagConstants.STRINGLIT ==> auxVal instanceof String) &&
116 (ttype==TagConstants.CHARLIT ==> auxVal instanceof Integer) &&
117 (ttype==TagConstants.LEXICALPRAGMA ==> auxVal instanceof
118 LexicalPragma) &&
119 (ttype==TagConstants.MODIFIERPRAGMA ==> auxVal instanceof
120 ModifierPragma) &&
121 (ttype==TagConstants.STMTPRAGMA ==> auxVal instanceof StmtPragma) &&
122 (ttype==TagConstants.TYPEDECLELEMPRAGMA ==> auxVal instanceof
123 TypeDeclElemPragma) &&
124 (ttype==TagConstants.TYPEMODIFIERPRAGMA ==> auxVal instanceof
125 TypeModifierPragma)
126 ); */
127 public Object auxVal;
128
129
130 /***************************************************
131 * *
132 * Creation: *
133 * *
134 **************************************************/
135
136 /*
137 * (This used to violate invariants ... but it should not any more.)
138 *
139 * NOTE: This is not a helper; we use invalid tokens in TokenQueue and
140 * Lex.savedState.
141 */
142 public Token() {}
143
144
145 /**
146 * Copy all the fields of <code>this</code> into
147 * <code>dst</code>. For convenience, returns <code>dst</code>.
148 */
149 /*@ ensures
150 dst.ttype == ttype &&
151 dst.startingLoc == startingLoc &&
152 dst.endingLoc == endingLoc &&
153 dst.identifierVal == identifierVal &&
154 dst.auxVal == auxVal &&
155 \result == dst;
156 */
157 public final /*@ non_null */ Token copyInto(/*@ non_null @*/ Token dst) {
158 dst.ttype = ttype;
159 dst.startingLoc = startingLoc;
160 dst.endingLoc = endingLoc;
161 dst.identifierVal = identifierVal;
162 dst.auxVal = auxVal;
163 return dst;
164 }
165
166
167 /***************************************************
168 * *
169 * Operations: *
170 * *
171 **************************************************/
172
173 /**
174 * Return a representation of <code>this</code> suitable for debug
175 * output.
176 */
177 public String ztoString() {
178 String result = Location.toFileName(startingLoc);
179 if (! Location.isWholeFileLoc(startingLoc))
180 result += ":" + Location.toOffset(startingLoc);
181 result += ": ";
182 if (ttype == TagConstants.IDENT)
183 result += "IDENT (" + identifierVal.toString() + ")";
184 else if (ttype == TagConstants.CHARLIT || ttype == TagConstants.INTLIT
185 || ttype == TagConstants.LONGLIT
186 || ttype == TagConstants.FLOATLIT
187 || ttype == TagConstants.DOUBLELIT
188 || ttype == TagConstants.STRINGLIT)
189 result += (PrettyPrint.inst.toString(ttype) + " ("
190 + PrettyPrint.toCanonicalString(ttype, auxVal) + ")");
191 else if (ttype == TagConstants.LEXICALPRAGMA
192 || ttype == TagConstants.MODIFIERPRAGMA
193 || ttype == TagConstants.TYPEDECLELEMPRAGMA
194 || ttype == TagConstants.STMTPRAGMA)
195 result += PrettyPrint.inst.toString(ttype) + " (" + auxVal.toString() + ")";
196 else result += PrettyPrint.inst.toString(ttype);
197 return result;
198 }
199
200
201 /** Check the invariants of <code>this</code>. */
202
203 public void zzz() {
204 Assert.notFalse(ttype != TagConstants.IDENT
205 || identifierVal != null);
206 Assert.notFalse(ttype != TagConstants.INTLIT
207 || auxVal instanceof Integer);
208 Assert.notFalse(ttype != TagConstants.LONGLIT
209 || auxVal instanceof Long);
210 Assert.notFalse(ttype != TagConstants.FLOATLIT
211 || auxVal instanceof Float);
212 Assert.notFalse(ttype != TagConstants.DOUBLELIT
213 || auxVal instanceof Double);
214 Assert.notFalse(ttype != TagConstants.STRINGLIT
215 || auxVal instanceof String);
216 Assert.notFalse(ttype != TagConstants.CHARLIT
217 || auxVal instanceof Integer);
218 Assert.notFalse(ttype != TagConstants.LEXICALPRAGMA
219 || auxVal == null || auxVal instanceof LexicalPragma);
220 Assert.notFalse(ttype != TagConstants.MODIFIERPRAGMA
221 || auxVal == null || auxVal instanceof ModifierPragma);
222 Assert.notFalse(ttype != TagConstants.STMTPRAGMA
223 || auxVal == null || auxVal instanceof StmtPragma);
224 Assert.notFalse(ttype != TagConstants.TYPEDECLELEMPRAGMA
225 || auxVal == null || auxVal instanceof TypeDeclElemPragma);
226 }
227 }