    001    package calhoun.util;
    002
    003    import java.io.BufferedOutputStream;
    004    import java.io.BufferedReader;
    005    import java.io.BufferedWriter;
    006    import java.io.File;
    007    import java.io.FileInputStream;
    008    import java.io.FileOutputStream;
    009    import java.io.FileReader;
    010    import java.io.FileWriter;
    011    import java.io.IOException;
    012    import java.io.InputStream;
    013    import java.io.InputStreamReader;
    014    import java.io.ObjectInputStream;
    015    import java.io.ObjectOutputStream;
    016    import java.io.OutputStream;
    017    import java.io.PrintWriter;
    018    import java.io.RandomAccessFile;
    019    import java.io.Reader;
    020    import java.io.Writer;
    021    import java.nio.channels.FileChannel;
    022    import java.nio.channels.FileLock;
    023    import java.util.ArrayList;
    024    import java.util.HashMap;
    025    import java.util.Iterator;
    026    import java.util.Map;
    027    import java.util.zip.GZIPInputStream;
    028    import java.util.zip.ZipEntry;
    029    import java.util.zip.ZipInputStream;
    030
    031    import org.apache.commons.logging.Log;
    032    import org.apache.commons.logging.LogFactory;
    033
    034    /** Utility functions for file management
    035     */
    036    public class FileUtil {
    037            private static final Log log = LogFactory.getLog(FileUtil.class);
    038
    039            /** This class should not be instantiated. */
    040            private FileUtil() {
    041            }
    042
    043            public static final String readFile(File filename) throws IOException {
    044                    return readReader(new FileReader(filename));
    045            }
    046
    047            public static final void appendSeparator(File in) throws Exception {
    048                    FileOutputStream f = new FileOutputStream(in, true);
    049                    f.write(java.io.File.separator.getBytes());
    050            }
    051            public static final void copyFile(File in, File out, boolean append) throws Exception {
    052                    FileChannel sourceChannel = new FileInputStream(in).getChannel();
    053                    FileChannel destinationChannel = new FileOutputStream(out, append).getChannel();
    054                    sourceChannel.transferTo(0, sourceChannel.size(), destinationChannel);
    055                    sourceChannel.close();
    056                    destinationChannel.close();
    057            }
    058            public static final void copyFile(File in, File out) throws Exception {
    059                    copyFile(in, out, false);
    060            }
    061
    062            public static final void renameFile(File oldFile, File newFile) throws Exception {
    063                    newFile.getParentFile().mkdirs();
    064                    if(!oldFile.renameTo(newFile)) {
    065                            copyFile(oldFile, newFile);
    066                            oldFile.delete();
    067                    }
    068            }
    069
    070            public static final BufferedWriter safeOpen(final String file) {
    071                    try {
    072                            return file == null ? null :
    073                                    new BufferedWriter(new FileWriter(file));
    074                    }
    075                    catch(IOException ex) {
    076                            throw new RuntimeException(ex);
    077                    }
    078            }
    079
    080            public static final void safeWrite(final Writer w, final String s) {
    081                    try {
    082                            w.write(s);
    083                    }
    084                    catch(IOException ex) {
    085                            throw new RuntimeException(ex);
    086                    }
    087            }
    088
    089            public static final void unzipFile(File file, File toDirectory) throws Exception {
    090                toDirectory.mkdirs();
    091                ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
    092                ZipEntry entry;
    093                while ( ( entry = zis.getNextEntry() ) != null ) {
    094                    if ( entry.isDirectory() )
    095                        new File(toDirectory, entry.getName()).mkdirs();
    096                    else {
    097                        FileOutputStream fos = new FileOutputStream(new File(toDirectory, entry.getName()));
    098                        byte[] bytes = new byte[8096];
    099                        int read;
    100                        while ( ( read = zis.read(bytes) ) != -1 ) {
    101                            fos.write(bytes, 0, read);
    102                        }
    103                    }
    104                }
    105            }
    106
    107            /**
    108             * Checks to see if the stream is in GZIP format, assuming that
    109             * it is open to the start of a file or other resource.  Note that
    110             * this advances the stream, caller should mark and reset around this method.
    111             */
    112            public static final boolean isGzipStream(InputStream s) {
    113                    try {
    114                            byte[] b = {0,0};
    115                            int magic = 0;
    116                            if (s.read(b) == 2) {
    117                                    magic = ((b[1]&0xff) << 8 | (b[0]&0xff));
    118                            }
    119                            return magic == GZIPInputStream.GZIP_MAGIC;
    120                    }
    121                    catch (IOException e) {
    122                            throw new ErrorException(e);
    123                    }
    124            }
    125
    126            public static final boolean isGzipFile(File file) {
    127                    try {
    128                            InputStream s = new FileInputStream(file);
    129                            boolean result = isGzipStream(s);
    130                            s.close();
    131                            return result;
    132                    }
    133                    catch (IOException e) {
    134                            throw new ErrorException(e);
    135                    }
    136            }
    137
    138            public static final boolean isGzipFile(String filename) {
    139                    return isGzipFile(new File(filename));
    140            }
    141
    142            public static final String[] getBaseAndExtension(File file) {
    143                    String name = file.getName();
    144                    int period = name.lastIndexOf('.');
    145                    if(period == -1) {
    146                            return new String[] { name, null };
    147                    }
    148                    else {
    149                            return new String[] { name.substring(0, period), name.substring(period+1) };
    150                    }
    151            }
    152
    153            public static File makeTempCopy(File in) throws Exception{
    154                    int index = 0;
    155                    String pathPrefix = in.getAbsolutePath();
    156                    File f;
    157                    do {
    158                            f = new File(pathPrefix + "temp_"+index);
    159                            index ++;
    160                    } while(f.exists());
    161
    162                    copyFile(in, f);
    163
    164                    return f;
    165            }
    166
    167            public static final String readInputStream(InputStream is) throws IOException {
    168                    BufferedReader reader = new BufferedReader(new InputStreamReader(is));
    169                    return readReader(reader);
    170            }
    171
    172            public static final String readReader(Reader reader) throws IOException {
    173                    char buf[] = new char[4096];
    174                    StringBuffer ret = new StringBuffer();
    175                    int size = 0;
    176                    do {
    177                            size = reader.read(buf);
    178                            if (size != -1)
    179                                    ret.append(buf, 0, size);
    180                    } while (size != -1);
    181                    return ret.toString();
    182            }
    183
    184            public static final byte[] readFileAsBytes(String filename) throws IOException {
    185                    File f = new File(filename);
    186                    byte[] data = new byte[(int) f.length()];
    187                    InputStream is = new FileInputStream(f);
    188                    is.read(data);
    189                    is.close();
    190                    return data;
    191            }
    192
    193            /** Reads a files into a String.
    194             *
    195             * @param filename  Name of the file to read in
    196             * @return String containing the contents of the file
    197             */
    198            public static final String readFile(String filename) throws IOException {
    199                    return readFile(new File(filename));
    200            }
    201
    202            public static String[][] readFlatFile(String fileName) throws IOException {
    203                    BufferedReader in = new BufferedReader(new FileReader(fileName));
    204                    ArrayList<String[]> ret = new ArrayList<String[]>();
    205                    String line;
    206                    while ((line = in.readLine()) != null) {
    207                            if(line.trim().charAt(0) == '#') {
    208                                    continue;
    209                            }
    210                            ret.add(line.split("\t"));
    211                    }
    212                    return (String[][]) ret.toArray(new String[ret.size()][]);
    213            }
    214
    215            public static String[][] readFlatFileWithComments(String fileName) throws IOException {
    216                    BufferedReader in = new BufferedReader(new FileReader(fileName));
    217                    ArrayList<String[]> ret = new ArrayList<String[]>();
    218                    String line;
    219                    while ((line = in.readLine()) != null) {
    220                            ret.add(line.split("\t"));
    221                    }
    222                    return (String[][]) ret.toArray(new String[ret.size()][]);
    223            }
    224
    225            public static double[] readDoublesFromSingleTabbedLine(String fileName) throws IOException {
    226                    String[][] s = FileUtil.readFlatFile(fileName);
    227                    Assert.a(s.length == 1);
    228                    int len = s[0].length;
    229                    double[] ret = new double[len];
    230                    for (int j=0; j<len; j++) { ret[j] = Double.parseDouble(s[0][j]) ; }
    231                    return ret;
    232            }
    233
    234            public static String[] readLines(String fileName) throws IOException {
    235                    BufferedReader in = new BufferedReader(new FileReader(fileName));
    236                    ArrayList ret = new ArrayList();
    237                    String line;
    238                    while ((line = in.readLine()) != null) {
    239                            ret.add(line);
    240                    }
    241                    return (String[]) ret.toArray(new String[ret.size()]);
    242            }
    243
    244            /** Writes a string out to a file.
    245             *
    246             * @param filename  Name of the file to read in
    247             * @param data The string to write to the file
    248             */
    249            public static final void writeFile(File filename, String data) throws IOException {
    250                    FileWriter fw = new FileWriter(filename);
    251                    PrintWriter logWriter = new PrintWriter(fw);
    252                    logWriter.print(data);
    253                    fw.close();
    254            }
    255
    256            /** Serializes a Java object out to a file.
    257             *
    258             * @param filename  Name of the file to write
    259             * @param data The object to serialize.  Must be Serializable
    260             */
    261            public static final void writeObject(String filename, Object data) throws IOException {
    262                    OutputStream fw = new BufferedOutputStream(new FileOutputStream(filename));
    263                    ObjectOutputStream objectWriter = new ObjectOutputStream(fw);
    264                    objectWriter.writeObject(data);
    265                    objectWriter.close();
    266            }
    267
    268            /** Read a serialized Java object from a file.
    269             *
    270             * @param filename  Name of the file to read in
    271             * @return The object which was deserialized.
    272             */
    273            public static final Object readObject(String filename) throws IOException, ClassNotFoundException {
    274                    FileInputStream fr = new FileInputStream(filename);
    275                    ObjectInputStream objectIn = new ObjectInputStream(fr);
    276                    Object object = objectIn.readObject();
    277                    objectIn.close();
    278                    return object;
    279            }
    280
    281            /** Compares records in two flat files.  Just checks that every line in one file occurs in the other.  Reports differences.
    282             * Like fileDiff, except that it is insensitive to differences in ordering.
    283             */
    284            public static final String fileRecordCompare(String expected, String actual, String maskingRegEx) throws IOException {
    285                    BufferedReader expectedReader = new BufferedReader(new FileReader(expected));
    286
    287                    Map records = new HashMap();
    288                    while (true) {
    289                            String s = expectedReader.readLine();
    290                            if (s == null)
    291                                    break;
    292                            if(maskingRegEx != null)
    293                                    s = s.replaceAll(maskingRegEx, "[!MASKED!]");
    294                            Number count = (Number)records.get(s);
    295                            if (count == null) records.put(s,new Integer(1));
    296                            else records.put(s,new Integer(count.intValue()+1));
    297                    }
    298                    expectedReader.close();
    299
    300                    StringBuffer ret = new StringBuffer();
    301                    BufferedReader actualReader = new BufferedReader(new FileReader(actual));
    302                    while (true) {
    303                            String s = actualReader.readLine();
    304                            if (s == null)
    305                                    break;
    306                            if(maskingRegEx != null)
    307                                    s = s.replaceAll(maskingRegEx, "[!MASKED!]");
    308                            if (records.containsKey(s)) {
    309                                    Number count = (Number)records.get(s);
    310                                    if (count.intValue() == 1) records.remove(s);
    311                                    else records.put(s,new Integer(count.intValue()-1));
    312                            } else {
    313                                    ret.append("+ " + s + "\n");
    314                            }
    315                    }
    316                    for (Iterator iter = records.keySet().iterator(); iter.hasNext();) {
    317                            String element = (String) iter.next();
    318                            ret.append("- " + element + "\n");
    319                    }
    320                    if (ret.length() == 0) {
    321                            return null;
    322                    } else {
    323                            return ret.toString();
    324                    }
    325            }
    326
    327            /** Convience form of fileRecordCompare which does no masking
    328             */
    329            public static final String fileRecordCompare(String expected, String actual) throws IOException {
    330                    return fileRecordCompare(expected, actual, null);
    331            }
    332
    333            /** Writes a string out to a file.
    334             *
    335             * @param filename  Name of the file to read in
    336             * @param data The string ot write to the file
    337             */
    338            public static final void writeFile(String filename, String data) throws IOException {
    339                    writeFile(new File(filename), data);
    340            }
    341
    342            /** Adds a file or directory name to an existing directory name.  Handles duplicate or missing slashes
    343             *
    344             * @param parent Name of the parent directory
    345             * @param child  Name of the child directory
    346             * @return The concatenated path.
    347             */
    348            public static final String appendPath(String parent, String child) {
    349                    // Use forward separator and not the system default.  Forward works on windows.
    350                    String separator = "/";
    351                    StringBuffer buf = new StringBuffer(parent);
    352                    if (!parent.endsWith(separator))
    353                            buf.append(separator);
    354                    if (child.startsWith(separator))
    355                            buf.append(child.substring(1));
    356                    else
    357                            buf.append(child);
    358                    return buf.toString();
    359            }
    360
    361            public static final void safeClose(Writer f) {
    362                    try {
    363                            if (f != null)
    364                                    f.close();
    365                    } catch (IOException ex) {
    366                            throw new ErrorException("Error closing writer: " + f, ex);
    367                    }
    368            }
    369
    370            public static final void safeClose(Reader f) {
    371                    try {
    372                            if (f != null)
    373                                    f.close();
    374                    } catch (IOException ex) {
    375                            throw new ErrorException("Error closing reader: " + f, ex);
    376                    }
    377            }
    378
    379            public static final void safeClose(OutputStream f) {
    380                    try {
    381                            if (f != null)
    382                                    f.close();
    383                    } catch (IOException ex) {
    384                            throw new ErrorException("Error closing outputStream: " + f, ex);
    385                    }
    386            }
    387
    388            public static final void safeClose(InputStream f) {
    389                    try {
    390                            if (f != null)
    391                                    f.close();
    392                    } catch (IOException ex) {
    393                            throw new ErrorException("Error closing inputStream: " + f, ex);
    394                    }
    395            }
    396
    397            public static final void safeClose(RandomAccessFile f) {
    398                    try {
    399                            if (f != null)
    400                                    f.close();
    401                    } catch (IOException ex) {
    402                            throw new ErrorException("Error closing randomAccessFile: " + f, ex);
    403                    }
    404            }
    405
    406            /**
    407             * Locks a file.
    408             *
    409             * This lock will work across JVMs but not necessarily across threads
    410             * within a single JVM. You must have write permission to lock a file.
    411             */
    412            public static final FileLock lockFile(RandomAccessFile raf) {
    413                    // given a RandomAccessFile object, gets the unique channel associated
    414                    // with it and locks it
    415                    FileChannel ch = raf.getChannel();
    416
    417                    // after the lock has been acquired, continue to use the raf object to write
    418                    // data to the File since RAF's implement different data output methods than the
    419                    // FileChannel class. The lock object will behave essentially as a mutex: if
    420                    // you have it, you've got the lock. If you don't, wait for it.
    421
    422                    // return the lock object - technically not necessary as the lock will
    423                    // be released when the raf is closed or when FileUtil.safeRelease gets called
    424                    // but it might be useful to have around
    425                    return doLock(ch);
    426            }
    427
    428            /**
    429             * Locks a file.
    430             *
    431             * This lock will work across JVMs but not necessarily across threads
    432             * within a single JVM. You must have write permission to lock a file.
    433             */
    434            public static final FileLock lockFile(FileOutputStream fos) {
    435                    FileChannel ch = fos.getChannel();
    436                    return doLock(ch);
    437            }
    438
    439            private static final FileLock doLock(FileChannel ch) {
    440                    FileLock lock;
    441                    try {
    442                            lock = ch.lock();
    443                    } catch (IOException ex) {
    444                            log.warn("File locking not supported on this platform. Continuing.");
    445                            log.warn("If the file " + ch + " is in use by another process, " +
    446                              "silent corruption may occur.");
    447                            lock = null;
    448                    } catch (Exception ex) {
    449                            throw new ErrorException("Error trying to acquire lock on file. ", ex);
    450                    }
    451                    return lock;
    452            }
    453
    454            public static final void safeRelease(FileLock lock) {
    455                    try {
    456                            if (lock != null)
    457                                    lock.release();
    458                    } catch (IOException ex) {
    459                            throw new ErrorException("Error releasing lock: " + lock, ex);
    460                    }
    461            }
    462
    463            /** Determines the size of a newline character in the given text file.  Will return 1 or 2 depending on the
    464             * platform the file was created on.
    465             * @param file the file object to examine.
    466             * @return the size of a newline in this file
    467             * @throws IOException
    468             */
    469            public static byte determineNewlineSize(File file) throws IOException {
    470                    InputStream stream = new FileInputStream(file);
    471                    try {
    472                            byte[] data = new byte[1000];
    473
    474                            int count;
    475                            do {
    476                                    count = stream.read(data);
    477                                    log.debug("byte read in " + file.getName() +" : " + count );
    478                                    for (int i = 0; i < count; ++i) {
    479                                            if (data[i] == 10) {
    480                                                    log.debug("Found Newline");
    481                                                    return 1;
    482                                            } else if (data[i] == 13) {
    483                                                    log.debug("Found CarriageReturn");
    484                                                    // Handle the edge case where 13 is the last character in the buffer
    485                                                    int nextChar = i + 1 < count ? data[i + 1] : stream.read();
    486                                                    return (byte) ((nextChar == 10) ? 2 : 1);
    487                                            }
    488                                    }
    489                            } while (count != -1);
    490                            // if File does not contain a newLine, we can safely assume that the size of a newline is 1.
    491                            return 1;
    492                            //throw new ConfigException("File did not contain a newline, could not determine type.");
    493                    } finally {
    494                            stream.close();
    495                    }
    496            }
    497    }