001 /*
002 * Copyright (C) 2000-2001 Iowa State University
003 *
004 * This file is part of mjc, the MultiJava Compiler.
005 *
006 * This program is free software; you can redistribute it and/or modify
007 * it under the terms of the GNU General Public License as published by
008 * the Free Software Foundation; either version 2 of the License, or
009 * (at your option) any later version.
010 *
011 * This program is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
014 * GNU General Public License for more details.
015 *
016 * You should have received a copy of the GNU General Public License
017 * along with this program; if not, write to the Free Software
018 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
019 *
020 * $Id: TestFilesTestSuite.java,v 1.11 2006/08/13 02:36:41 chalin Exp $
021 * Author: David R. Cok
022 */
023 package junitutils;
024 import junit.framework.*;
025 import java.io.*;
026 import java.util.Iterator;
027 import java.lang.reflect.Method;
028
029 /** This is a JUnit TestSuite that is created from a number of tests as follows.
030 Each TestCase is an instance of the inner class Helper, instantiated with
031 a name of a file. The file names are read from the file named by the
032 parameter 'fileOfTestFilenames'. The argument to the constructor named
033 'args' provides a set of command-line arguments; the filename for the TestCase
034 is added on to the end of the list of command-line arguments. Then the static
035 compile method of the given class is called on those command-line arguments.
036 <P>
037 The standard output and error output is captured from the execution of the
038 compile method. This is compared to the output in filename + "-expected".
039 The TestCase succeeds if these match; if they do not match, the test fails and
040 the actual output is saved in filename + "-ckd".
041 <P>
042 The test must be run from the directory in which it resides - because
043 it creates and opens files in the current directory.
044
045 @author David R. Cok
046 */
047 public class TestFilesTestSuite extends TestSuite {
048
049 //@ ensures_redundantly !initialized;
050 protected TestFilesTestSuite() {}
051
052 /*
053 Create derived classes with alternate tests by deriving a class from this
054 one. It should contain an inner Helper class derived from
055 TestFilesTestSuite.Helper. The method TestFilesTestSuite.makeHelper
056 should be overridden to return an instance of the derived Helper class.
057 The derived Helper class should override Helper.dotest to do the actual
058 test.
059 */
060
061 // -------------------------------------------------------------
062 // DATA MEMBERS
063 // -------------------------------------------------------------
064
065 /** The name of this test suite. */
066 protected String testName; //@ in initialized;
067 //@ protected invariant_redundantly initialized ==> (testName != null);
068
069 /** The method that is to be executed on the command-line arguments. */
070 protected Method method; //@ in initialized;
071 //@ protected invariant_redundantly initialized ==> (method != null);
072
073 final static String SAVED_SUFFIX = "-ckd";
074 final static String ORACLE_SUFFIX = "-expected";
075
076 // -------------------------------------------------------------
077 // CONSTRUCTOR
078 // -------------------------------------------------------------
079
080 /** A constructor for this test suite.
081 @param testName The name of the test suite
082 @param fileOfTestFilenames The file to be read for filenames of tests
083 @param args The command-line arguments that the static compile
084 method will be applied to, with the filename added on
085 @param cls The class in which to find the static compile method
086 */
087 /*@ public behavior
088 @ assignable initialized, objectState;
089 @ ensures_redundantly initialized;
090 @ signals_only RuntimeException;
091 @*/
092 public TestFilesTestSuite(/*@ non_null */ String testName,
093 /*@ non_null */ String fileOfTestFilenames,
094 String[] args, // Ignored! FIXME
095 /*@ non_null */ Class cls
096 ) {
097 super(testName);
098 this.testName = testName;
099 try {
100 method = cls.getMethod("compile", new Class[] { String[].class });
101 Class rt = method.getReturnType();
102 if(rt != Integer.TYPE && rt != Boolean.TYPE) {
103 String m = (cls.getName() +
104 ".compile() must have a return type of " +
105 "'int' or 'boolean', not " + rt);
106 throw new RuntimeException(m);
107 }
108 } catch (NoSuchMethodException e) {
109 throw new RuntimeException(e.toString());
110 }
111
112 try {
113 Iterator i = new LineIterator(fileOfTestFilenames);
114 while (i.hasNext()) {
115 String s = (String)i.next();
116 String[] allargs = Utils.parseLine(s);
117 if(allargs.length > 0) {
118 s = allargs[allargs.length-1];
119 addTest(makeHelper(s,allargs));
120 }
121 }
122 } catch (java.io.IOException e) {
123 throw new RuntimeException(e.toString());
124 }
125 }
126
127 //@ public model boolean initialized;
128 /*@ protected represents initialized <- testName != null
129 @ && method != null
130 @ && (method.getReturnType() == int.class
131 @ || method.getReturnType() == boolean.class);
132 @*/
133
134 //@ ensures \result == initialized;
135 private /*@ pure */ boolean initialized() {
136 return testName != null
137 && method != null
138 && (method.getReturnType() == int.class
139 || method.getReturnType() == boolean.class);
140 }
141
142 /** Factory method for the helper class object. */
143 //@ assignable \nothing;
144 protected Helper makeHelper(/*@ non_null */ String filename,
145 /*@ non_null */ String[] args)
146 {
147 return new Helper(filename,args);
148 }
149
150
151 // FIXME - This test does not do the equivalent of FIXTILT or PATHTOFILES
152 // that is performed in the Makefile to canonicalize the outputs. So far we
153 // have not needed it.
154
155 /** This is a helper class that is actually a TestCase; it is run repeatedly
156 with different constructor arguments.
157 */
158 public class Helper extends TestCase {
159
160 /**
161 The first argument is used as the name of the test as well as
162 the name of the file to be tested.
163 */
164 public Helper(/*@ non_null */ String testname, /*@ non_null */ String[] args) {
165 super(testname);
166 this.fileToTest = testname;
167 this.args = args;
168 }
169
170 /** Filename of comparison files */
171 protected /*@ non_null */ String fileToTest;
172
173 /** Result of test */
174 protected /*@ nullable */ Object returnedObject;
175
176 /** Command-line arguments (including filename) for this test. */
177 protected /*@ non_null */ String[] args;
178
179 /** This is the framework around the test. It sets up the streams to
180 capture output, and catches all relevant exceptions.
181 */
182
183 //@ also
184 //@ requires initialized;
185 // Maybe we could move the above spec case into the superclass?
186 public void runTest() throws java.io.IOException {
187 // Due to behavioral subtyping this method might be called
188 // when !initialized ... hence we will test for this condition
189 if(!initialized()) {
190 String msg = "TestFilesTestSuite.runTest() "
191 + "called before 'this' was properly initialized";
192 fail(msg);
193 }
194
195 //System.out.println("\nTest suite " + testName + ": " + fileToTest);
196 //for (int kk=0; kk<args.length; ++kk) System.out.println(args[kk]);
197 //System.out.println();
198
199 ByteArrayOutputStream ba = Utils.setStreams();
200 try {
201 returnedObject = dotest(fileToTest,args);
202 } catch (IllegalAccessException e) {
203 Utils.restoreStreams(true);
204 fail(e.toString());
205 } catch (IllegalArgumentException e) {
206 Utils.restoreStreams(true);
207 fail(e.toString());
208 } catch (java.lang.reflect.InvocationTargetException e) {
209 Utils.restoreStreams(true);
210 java.io.StringWriter sw = new StringWriter();
211 sw.write(e.toString());
212 e.printStackTrace(new PrintWriter(sw));
213 fail(sw.toString());
214 /* } catch (Throwable e) { // THIS JUST FOR DEBUG
215 Utils.restoreStreams(true); // must restore before use of System.out on the next line
216 System.out.println(e);
217 e.printStackTrace();
218 */
219 } finally {
220 Utils.restoreStreams(true);
221 }
222 /*@ nullable */ String err = doOutputCheck(fileToTest,ba.toString(),returnedObject);
223 if (err != null) fail(err);
224 //System.out.println("COMPLETED: " + fileToTest);
225 }
226 } // end of class Helper
227
228 /** This is the actual test; it compiles the given file and compares its
229 output to the expected result (in fileToTest+ORACLE_SUFFIX); the
230 output is expected to
231 match and the result of the compile to be true or false, depending on
232 whether errors or warnings were reported. Override this method in derived tests.
233
234 */
235 //@ requires initialized;
236 protected /*@ non_null */ Object dotest(String fileToTest, String[] args)
237 throws IllegalAccessException, IllegalArgumentException,
238 java.lang.reflect.InvocationTargetException
239 {
240 return method.invoke(null,new Object[]{args});
241 }
242
243
244 //@ requires initialized;
245 protected /*@ nullable */ String doOutputCheck(/*@ non_null */ String fileToTest,
246 /*@ non_null */ String output,
247 /*@ non_null */ Object returnedValue) {
248 try {
249 String expectedOutput = Utils.readFile(fileToTest+ORACLE_SUFFIX);
250 Diff df = new Diff("expected", expectedOutput, "actual", output);
251
252 if (!df.areDifferent()) {
253 // If the two strings match, the test succeeds and we make sure
254 // that there is no -ckd file to confuse anyone.
255 (new File(fileToTest+SAVED_SUFFIX)).delete();
256 } else {
257 // If the strings do not match, we save the actual string and
258 // fail the test.
259 FileWriter f = null;
260 try {
261 f = new FileWriter(fileToTest+SAVED_SUFFIX);
262 f.write(output);
263 } finally {
264 if (f != null) f.close();
265 }
266
267 return (df.result());
268 }
269 return checkReturnValue(fileToTest,expectedOutput,returnedValue);
270 } catch (java.io.IOException e) {
271 return (e.toString());
272 }
273 }
274
275 //@ requires initialized;
276 public /*@ nullable */ String checkReturnValue(/*@ non_null */ String fileToTest,
277 /*@ non_null */ String expectedOutput,
278 /*@ non_null */ Object returnedValue)
279 {
280 if (returnedValue instanceof Boolean) {
281 return expectedStatusReport(fileToTest,
282 ((Boolean)returnedValue).booleanValue(),
283 expectedOutput);
284 } else if (returnedValue instanceof Integer) {
285 return expectedStatusReport(fileToTest,
286 ((Integer)returnedValue).intValue(),
287 expectedOutput);
288 } else {
289 return ("The return value is of type " + returnedValue.getClass()
290 + " instead of int or boolean");
291 }
292 }
293
294
295 /** Returns null if ok, otherwise returns failure message. */
296 //@ requires initialized;
297 public /*@ nullable */ String expectedStatusReport(/*@ non_null */ String fileToTest,
298 int ecode,
299 /*@ non_null */ String expectedOutput)
300 {
301 int ret = expectedIntegerStatus(fileToTest,expectedOutput);
302 if (ecode == ret) return null;
303 return "The compile produced an invalid return value. It should be " + ret + " but instead is " + ecode;
304 }
305
306 //@ requires initialized;
307 public /*@ nullable */ String expectedStatusReport(/*@ non_null */ String fileToTest,
308 boolean b,
309 /*@ non_null */ String expectedOutput)
310 {
311 boolean status = expectedBooleanStatus(fileToTest,expectedOutput);
312 if (status == b) return null;
313 return ("The compile produced an invalid return value. It should be "
314 + (!b) + " since there was " +
315 (b?"no ":"") + "error output but instead is " + b);
316 }
317
318 //@ requires initialized;
319 public boolean expectedBooleanStatus(/*@ non_null */ String fileToTest,
320 /*@ non_null */ String expectedOutput)
321 {
322 return expectedOutput.length()==0;
323 }
324
325 //@ requires initialized;
326 public int expectedIntegerStatus(/*@ non_null */ String fileToTest,
327 /*@ non_null */ String expectedOutput)
328 {
329 return 0;
330 }
331
332 }
333
334