001 package calhoun.util;
002
003 import java.io.BufferedOutputStream;
004 import java.io.BufferedReader;
005 import java.io.BufferedWriter;
006 import java.io.File;
007 import java.io.FileInputStream;
008 import java.io.FileOutputStream;
009 import java.io.FileReader;
010 import java.io.FileWriter;
011 import java.io.IOException;
012 import java.io.InputStream;
013 import java.io.InputStreamReader;
014 import java.io.ObjectInputStream;
015 import java.io.ObjectOutputStream;
016 import java.io.OutputStream;
017 import java.io.PrintWriter;
018 import java.io.RandomAccessFile;
019 import java.io.Reader;
020 import java.io.Writer;
021 import java.nio.channels.FileChannel;
022 import java.nio.channels.FileLock;
023 import java.util.ArrayList;
024 import java.util.HashMap;
025 import java.util.Iterator;
026 import java.util.Map;
027 import java.util.zip.GZIPInputStream;
028 import java.util.zip.ZipEntry;
029 import java.util.zip.ZipInputStream;
030
031 import org.apache.commons.logging.Log;
032 import org.apache.commons.logging.LogFactory;
033
034 /** Utility functions for file management
035 */
036 public class FileUtil {
037 private static final Log log = LogFactory.getLog(FileUtil.class);
038
039 /** This class should not be instantiated. */
040 private FileUtil() {
041 }
042
043 public static final String readFile(File filename) throws IOException {
044 return readReader(new FileReader(filename));
045 }
046
047 public static final void appendSeparator(File in) throws Exception {
048 FileOutputStream f = new FileOutputStream(in, true);
049 f.write(java.io.File.separator.getBytes());
050 }
051 public static final void copyFile(File in, File out, boolean append) throws Exception {
052 FileChannel sourceChannel = new FileInputStream(in).getChannel();
053 FileChannel destinationChannel = new FileOutputStream(out, append).getChannel();
054 sourceChannel.transferTo(0, sourceChannel.size(), destinationChannel);
055 sourceChannel.close();
056 destinationChannel.close();
057 }
058 public static final void copyFile(File in, File out) throws Exception {
059 copyFile(in, out, false);
060 }
061
062 public static final void renameFile(File oldFile, File newFile) throws Exception {
063 newFile.getParentFile().mkdirs();
064 if(!oldFile.renameTo(newFile)) {
065 copyFile(oldFile, newFile);
066 oldFile.delete();
067 }
068 }
069
070 public static final BufferedWriter safeOpen(final String file) {
071 try {
072 return file == null ? null :
073 new BufferedWriter(new FileWriter(file));
074 }
075 catch(IOException ex) {
076 throw new RuntimeException(ex);
077 }
078 }
079
080 public static final void safeWrite(final Writer w, final String s) {
081 try {
082 w.write(s);
083 }
084 catch(IOException ex) {
085 throw new RuntimeException(ex);
086 }
087 }
088
089 public static final void unzipFile(File file, File toDirectory) throws Exception {
090 toDirectory.mkdirs();
091 ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
092 ZipEntry entry;
093 while ( ( entry = zis.getNextEntry() ) != null ) {
094 if ( entry.isDirectory() )
095 new File(toDirectory, entry.getName()).mkdirs();
096 else {
097 FileOutputStream fos = new FileOutputStream(new File(toDirectory, entry.getName()));
098 byte[] bytes = new byte[8096];
099 int read;
100 while ( ( read = zis.read(bytes) ) != -1 ) {
101 fos.write(bytes, 0, read);
102 }
103 }
104 }
105 }
106
107 /**
108 * Checks to see if the stream is in GZIP format, assuming that
109 * it is open to the start of a file or other resource. Note that
110 * this advances the stream, caller should mark and reset around this method.
111 */
112 public static final boolean isGzipStream(InputStream s) {
113 try {
114 byte[] b = {0,0};
115 int magic = 0;
116 if (s.read(b) == 2) {
117 magic = ((b[1]&0xff) << 8 | (b[0]&0xff));
118 }
119 return magic == GZIPInputStream.GZIP_MAGIC;
120 }
121 catch (IOException e) {
122 throw new ErrorException(e);
123 }
124 }
125
126 public static final boolean isGzipFile(File file) {
127 try {
128 InputStream s = new FileInputStream(file);
129 boolean result = isGzipStream(s);
130 s.close();
131 return result;
132 }
133 catch (IOException e) {
134 throw new ErrorException(e);
135 }
136 }
137
138 public static final boolean isGzipFile(String filename) {
139 return isGzipFile(new File(filename));
140 }
141
142 public static final String[] getBaseAndExtension(File file) {
143 String name = file.getName();
144 int period = name.lastIndexOf('.');
145 if(period == -1) {
146 return new String[] { name, null };
147 }
148 else {
149 return new String[] { name.substring(0, period), name.substring(period+1) };
150 }
151 }
152
153 public static File makeTempCopy(File in) throws Exception{
154 int index = 0;
155 String pathPrefix = in.getAbsolutePath();
156 File f;
157 do {
158 f = new File(pathPrefix + "temp_"+index);
159 index ++;
160 } while(f.exists());
161
162 copyFile(in, f);
163
164 return f;
165 }
166
167 public static final String readInputStream(InputStream is) throws IOException {
168 BufferedReader reader = new BufferedReader(new InputStreamReader(is));
169 return readReader(reader);
170 }
171
172 public static final String readReader(Reader reader) throws IOException {
173 char buf[] = new char[4096];
174 StringBuffer ret = new StringBuffer();
175 int size = 0;
176 do {
177 size = reader.read(buf);
178 if (size != -1)
179 ret.append(buf, 0, size);
180 } while (size != -1);
181 return ret.toString();
182 }
183
184 public static final byte[] readFileAsBytes(String filename) throws IOException {
185 File f = new File(filename);
186 byte[] data = new byte[(int) f.length()];
187 InputStream is = new FileInputStream(f);
188 is.read(data);
189 is.close();
190 return data;
191 }
192
193 /** Reads a files into a String.
194 *
195 * @param filename Name of the file to read in
196 * @return String containing the contents of the file
197 */
198 public static final String readFile(String filename) throws IOException {
199 return readFile(new File(filename));
200 }
201
202 public static String[][] readFlatFile(String fileName) throws IOException {
203 BufferedReader in = new BufferedReader(new FileReader(fileName));
204 ArrayList<String[]> ret = new ArrayList<String[]>();
205 String line;
206 while ((line = in.readLine()) != null) {
207 if(line.trim().charAt(0) == '#') {
208 continue;
209 }
210 ret.add(line.split("\t"));
211 }
212 return (String[][]) ret.toArray(new String[ret.size()][]);
213 }
214
215 public static String[][] readFlatFileWithComments(String fileName) throws IOException {
216 BufferedReader in = new BufferedReader(new FileReader(fileName));
217 ArrayList<String[]> ret = new ArrayList<String[]>();
218 String line;
219 while ((line = in.readLine()) != null) {
220 ret.add(line.split("\t"));
221 }
222 return (String[][]) ret.toArray(new String[ret.size()][]);
223 }
224
225 public static double[] readDoublesFromSingleTabbedLine(String fileName) throws IOException {
226 String[][] s = FileUtil.readFlatFile(fileName);
227 Assert.a(s.length == 1);
228 int len = s[0].length;
229 double[] ret = new double[len];
230 for (int j=0; j<len; j++) { ret[j] = Double.parseDouble(s[0][j]) ; }
231 return ret;
232 }
233
234 public static String[] readLines(String fileName) throws IOException {
235 BufferedReader in = new BufferedReader(new FileReader(fileName));
236 ArrayList ret = new ArrayList();
237 String line;
238 while ((line = in.readLine()) != null) {
239 ret.add(line);
240 }
241 return (String[]) ret.toArray(new String[ret.size()]);
242 }
243
244 /** Writes a string out to a file.
245 *
246 * @param filename Name of the file to read in
247 * @param data The string to write to the file
248 */
249 public static final void writeFile(File filename, String data) throws IOException {
250 FileWriter fw = new FileWriter(filename);
251 PrintWriter logWriter = new PrintWriter(fw);
252 logWriter.print(data);
253 fw.close();
254 }
255
256 /** Serializes a Java object out to a file.
257 *
258 * @param filename Name of the file to write
259 * @param data The object to serialize. Must be Serializable
260 */
261 public static final void writeObject(String filename, Object data) throws IOException {
262 OutputStream fw = new BufferedOutputStream(new FileOutputStream(filename));
263 ObjectOutputStream objectWriter = new ObjectOutputStream(fw);
264 objectWriter.writeObject(data);
265 objectWriter.close();
266 }
267
268 /** Read a serialized Java object from a file.
269 *
270 * @param filename Name of the file to read in
271 * @return The object which was deserialized.
272 */
273 public static final Object readObject(String filename) throws IOException, ClassNotFoundException {
274 FileInputStream fr = new FileInputStream(filename);
275 ObjectInputStream objectIn = new ObjectInputStream(fr);
276 Object object = objectIn.readObject();
277 objectIn.close();
278 return object;
279 }
280
281 /** Compares records in two flat files. Just checks that every line in one file occurs in the other. Reports differences.
282 * Like fileDiff, except that it is insensitive to differences in ordering.
283 */
284 public static final String fileRecordCompare(String expected, String actual, String maskingRegEx) throws IOException {
285 BufferedReader expectedReader = new BufferedReader(new FileReader(expected));
286
287 Map records = new HashMap();
288 while (true) {
289 String s = expectedReader.readLine();
290 if (s == null)
291 break;
292 if(maskingRegEx != null)
293 s = s.replaceAll(maskingRegEx, "[!MASKED!]");
294 Number count = (Number)records.get(s);
295 if (count == null) records.put(s,new Integer(1));
296 else records.put(s,new Integer(count.intValue()+1));
297 }
298 expectedReader.close();
299
300 StringBuffer ret = new StringBuffer();
301 BufferedReader actualReader = new BufferedReader(new FileReader(actual));
302 while (true) {
303 String s = actualReader.readLine();
304 if (s == null)
305 break;
306 if(maskingRegEx != null)
307 s = s.replaceAll(maskingRegEx, "[!MASKED!]");
308 if (records.containsKey(s)) {
309 Number count = (Number)records.get(s);
310 if (count.intValue() == 1) records.remove(s);
311 else records.put(s,new Integer(count.intValue()-1));
312 } else {
313 ret.append("+ " + s + "\n");
314 }
315 }
316 for (Iterator iter = records.keySet().iterator(); iter.hasNext();) {
317 String element = (String) iter.next();
318 ret.append("- " + element + "\n");
319 }
320 if (ret.length() == 0) {
321 return null;
322 } else {
323 return ret.toString();
324 }
325 }
326
327 /** Convience form of fileRecordCompare which does no masking
328 */
329 public static final String fileRecordCompare(String expected, String actual) throws IOException {
330 return fileRecordCompare(expected, actual, null);
331 }
332
333 /** Writes a string out to a file.
334 *
335 * @param filename Name of the file to read in
336 * @param data The string ot write to the file
337 */
338 public static final void writeFile(String filename, String data) throws IOException {
339 writeFile(new File(filename), data);
340 }
341
342 /** Adds a file or directory name to an existing directory name. Handles duplicate or missing slashes
343 *
344 * @param parent Name of the parent directory
345 * @param child Name of the child directory
346 * @return The concatenated path.
347 */
348 public static final String appendPath(String parent, String child) {
349 // Use forward separator and not the system default. Forward works on windows.
350 String separator = "/";
351 StringBuffer buf = new StringBuffer(parent);
352 if (!parent.endsWith(separator))
353 buf.append(separator);
354 if (child.startsWith(separator))
355 buf.append(child.substring(1));
356 else
357 buf.append(child);
358 return buf.toString();
359 }
360
361 public static final void safeClose(Writer f) {
362 try {
363 if (f != null)
364 f.close();
365 } catch (IOException ex) {
366 throw new ErrorException("Error closing writer: " + f, ex);
367 }
368 }
369
370 public static final void safeClose(Reader f) {
371 try {
372 if (f != null)
373 f.close();
374 } catch (IOException ex) {
375 throw new ErrorException("Error closing reader: " + f, ex);
376 }
377 }
378
379 public static final void safeClose(OutputStream f) {
380 try {
381 if (f != null)
382 f.close();
383 } catch (IOException ex) {
384 throw new ErrorException("Error closing outputStream: " + f, ex);
385 }
386 }
387
388 public static final void safeClose(InputStream f) {
389 try {
390 if (f != null)
391 f.close();
392 } catch (IOException ex) {
393 throw new ErrorException("Error closing inputStream: " + f, ex);
394 }
395 }
396
397 public static final void safeClose(RandomAccessFile f) {
398 try {
399 if (f != null)
400 f.close();
401 } catch (IOException ex) {
402 throw new ErrorException("Error closing randomAccessFile: " + f, ex);
403 }
404 }
405
406 /**
407 * Locks a file.
408 *
409 * This lock will work across JVMs but not necessarily across threads
410 * within a single JVM. You must have write permission to lock a file.
411 */
412 public static final FileLock lockFile(RandomAccessFile raf) {
413 // given a RandomAccessFile object, gets the unique channel associated
414 // with it and locks it
415 FileChannel ch = raf.getChannel();
416
417 // after the lock has been acquired, continue to use the raf object to write
418 // data to the File since RAF's implement different data output methods than the
419 // FileChannel class. The lock object will behave essentially as a mutex: if
420 // you have it, you've got the lock. If you don't, wait for it.
421
422 // return the lock object - technically not necessary as the lock will
423 // be released when the raf is closed or when FileUtil.safeRelease gets called
424 // but it might be useful to have around
425 return doLock(ch);
426 }
427
428 /**
429 * Locks a file.
430 *
431 * This lock will work across JVMs but not necessarily across threads
432 * within a single JVM. You must have write permission to lock a file.
433 */
434 public static final FileLock lockFile(FileOutputStream fos) {
435 FileChannel ch = fos.getChannel();
436 return doLock(ch);
437 }
438
439 private static final FileLock doLock(FileChannel ch) {
440 FileLock lock;
441 try {
442 lock = ch.lock();
443 } catch (IOException ex) {
444 log.warn("File locking not supported on this platform. Continuing.");
445 log.warn("If the file " + ch + " is in use by another process, " +
446 "silent corruption may occur.");
447 lock = null;
448 } catch (Exception ex) {
449 throw new ErrorException("Error trying to acquire lock on file. ", ex);
450 }
451 return lock;
452 }
453
454 public static final void safeRelease(FileLock lock) {
455 try {
456 if (lock != null)
457 lock.release();
458 } catch (IOException ex) {
459 throw new ErrorException("Error releasing lock: " + lock, ex);
460 }
461 }
462
463 /** Determines the size of a newline character in the given text file. Will return 1 or 2 depending on the
464 * platform the file was created on.
465 * @param file the file object to examine.
466 * @return the size of a newline in this file
467 * @throws IOException
468 */
469 public static byte determineNewlineSize(File file) throws IOException {
470 InputStream stream = new FileInputStream(file);
471 try {
472 byte[] data = new byte[1000];
473
474 int count;
475 do {
476 count = stream.read(data);
477 log.debug("byte read in " + file.getName() +" : " + count );
478 for (int i = 0; i < count; ++i) {
479 if (data[i] == 10) {
480 log.debug("Found Newline");
481 return 1;
482 } else if (data[i] == 13) {
483 log.debug("Found CarriageReturn");
484 // Handle the edge case where 13 is the last character in the buffer
485 int nextChar = i + 1 < count ? data[i + 1] : stream.read();
486 return (byte) ((nextChar == 10) ? 2 : 1);
487 }
488 }
489 } while (count != -1);
490 // if File does not contain a newLine, we can safely assume that the size of a newline is 1.
491 return 1;
492 //throw new ConfigException("File did not contain a newline, could not determine type.");
493 } finally {
494 stream.close();
495 }
496 }
497 }