View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements. See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache license, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License. You may obtain a copy of the License at
    8    *
    9    *      http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the license for the specific language governing permissions and
    15   * limitations under the license.
    16   */
    17  package org.apache.logging.log4j.core.appender;
    18
    19  import java.io.File;
    20  import java.io.IOException;
    21  import java.io.OutputStream;
    22  import java.io.RandomAccessFile;
    23  import java.io.Serializable;
    24  import java.lang.reflect.Method;
    25  import java.nio.ByteOrder;
    26  import java.nio.MappedByteBuffer;
    27  import java.nio.channels.FileChannel;
    28  import java.security.AccessController;
    29  import java.security.PrivilegedActionException;
    30  import java.security.PrivilegedExceptionAction;
    31  import java.util.HashMap;
    32  import java.util.Map;
    33
    34  import org.apache.logging.log4j.core.Layout;
    35  import org.apache.logging.log4j.core.util.Assert;
    36  import org.apache.logging.log4j.core.util.Closer;
    37  import org.apache.logging.log4j.core.util.NullOutputStream;
    38
    39  /**
    40   * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
    41   * memory and writes to this memory region.
    42   * <p>
    43   *
    44   * @see <a
    45   *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
    46   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
    47   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
    48   * @see <a
    49   *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
    50   *
    51   * @since 2.1
    52   */
    53  public class MemoryMappedFileManager extends OutputStreamManager {
    54      static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
    55      private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
    56
    57      private final boolean isForce;
    58      private final int regionLength;
    59      private final String advertiseURI;
    60      private final RandomAccessFile randomAccessFile;
    61      private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
    62      private MappedByteBuffer mappedBuffer;
    63      private long mappingOffset;
    64
    65      protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
    66              final boolean force, final long position, final int regionLength, final String advertiseURI,
    67              final Layout<? extends Serializable> layout) throws IOException {
    68          super(os, fileName, layout);
    69          this.isForce = force;
    70          this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
    71          this.regionLength = regionLength;
    72          this.advertiseURI = advertiseURI;
    73          this.isEndOfBatch.set(Boolean.FALSE);
    74          this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
    75          this.mappingOffset = position;
    76      }
    77
    78      /**
    79       * Returns the MemoryMappedFileManager.
    80       *
    81       * @param fileName The name of the file to manage.
    82       * @param append true if the file should be appended to, false if it should be overwritten.
    83       * @param isForce true if the contents should be flushed to disk on every write
    84       * @param regionLength The mapped region length.
    85       * @param advertiseURI the URI to use when advertising the file
    86       * @param layout The layout.
    87       * @return A MemoryMappedFileManager for the File.
    88       */
    89      public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
    90              final boolean isForce, final int regionLength, final String advertiseURI,
    91              final Layout<? extends Serializable> layout) {
    92          return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
    93                  advertiseURI, layout), FACTORY);
    94      }
    95
    96      public Boolean isEndOfBatch() {
    97          return isEndOfBatch.get();
    98      }
    99
    100     public void setEndOfBatch(final boolean isEndOfBatch) {
    101         this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
    102     }
    103
    104     @Override
    105     protected synchronized void write(final byte[] bytes, int offset, int length) {
    106         super.write(bytes, offset, length); // writes to dummy output stream
    107
    108         while (length > mappedBuffer.remaining()) {
    109             final int chunk = mappedBuffer.remaining();
    110             mappedBuffer.put(bytes, offset, chunk);
    111             offset += chunk;
    112             length -= chunk;
    113             remap();
    114         }
    115         mappedBuffer.put(bytes, offset, length);
    116
    117         // no need to call flush() if force is true,
    118         // already done in AbstractOutputStreamAppender.append
    119     }
    120
    121     private synchronized void remap() {
    122         final long offset = this.mappingOffset + mappedBuffer.position();
    123         final int length = mappedBuffer.remaining() + regionLength;
    124         try {
    125             unsafeUnmap(mappedBuffer);
    126             final long fileLength = randomAccessFile.length() + regionLength;
    127             LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
    128
    129             long startNanos = System.nanoTime();
    130             randomAccessFile.setLength(fileLength);
    131             final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    132             LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
    133
    134             mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
    135             mappingOffset = offset;
    136         } catch (final Exception ex) {
    137             LOGGER.error("Unable to remap " + getName() + ". " + ex);
    138         }
    139     }
    140
    141     @Override
    142     public synchronized void flush() {
    143         mappedBuffer.force();
    144     }
    145
    146     @Override
    147     public synchronized void close() {
    148         final long position = mappedBuffer.position();
    149         final long length = mappingOffset + position;
    150         try {
    151             unsafeUnmap(mappedBuffer);
    152         } catch (final Exception ex) {
    153             LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
    154         }
    155         try {
    156             LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
    157                     length, mappingOffset, position);
    158             randomAccessFile.setLength(length);
    159             randomAccessFile.close();
    160         } catch (final IOException ex) {
    161             LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
    162         }
    163     }
    164
    165     public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
    166             final int size) throws IOException {
    167         for (int i = 1;; i++) {
    168             try {
    169                 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
    170
    171                 final long startNanos = System.nanoTime();
    172                 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
    173                 map.order(ByteOrder.nativeOrder());
    174
    175                 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    176                 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
    177
    178                 return map;
    179             } catch (final IOException e) {
    180                 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
    181                     throw e;
    182                 }
    183                 LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
    184                 if (i < 10) {
    185                     Thread.yield();
    186                 } else {
    187                     try {
    188                         Thread.sleep(1);
    189                     } catch (final InterruptedException ignored) {
    190                         Thread.currentThread().interrupt();
    191                         throw e;
    192                     }
    193                 }
    194             }
    195         }
    196     }
    197
    198     private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
    199         LOGGER.debug("MMapAppender unmapping old buffer...");
    200         final long startNanos = System.nanoTime();
    201         AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
    202             @Override
    203             public Object run() throws Exception {
    204                 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
    205                 getCleanerMethod.setAccessible(true);
    206                 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
    207                 final Method cleanMethod = cleaner.getClass().getMethod("clean");
    208                 cleanMethod.invoke(cleaner);
    209                 return null;
    210             }
    211         });
    212         final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    213         LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
    214     }
    215
    216     /**
    217      * Returns the name of the File being managed.
    218      *
    219      * @return The name of the File being managed.
    220      */
    221     public String getFileName() {
    222         return getName();
    223     }
    224
    225     /**
    226      * Returns the length of the memory mapped region.
    227      *
    228      * @return the length of the mapped region
    229      */
    230     public int getRegionLength() {
    231         return regionLength;
    232     }
    233
    234     /**
    235      * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
    236      * {@code false} otherwise.
    237      *
    238      * @return whether each write should be force-sync'ed
    239      */
    240     public boolean isImmediateFlush() {
    241         return isForce;
    242     }
    243
    244     /**
    245      * Gets this FileManager's content format specified by:
    246      * <p>
    247      * Key: "fileURI" Value: provided "advertiseURI" param.
    248      * </p>
    249      *
    250      * @return Map of content format keys supporting FileManager
    251      */
    252     @Override
    253     public Map<String, String> getContentFormat() {
    254         final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
    255         result.put("fileURI", advertiseURI);
    256         return result;
    257     }
    258
    259     /**
    260      * Factory Data.
    261      */
    262     private static class FactoryData {
    263         private final boolean append;
    264         private final boolean force;
    265         private final int regionLength;
    266         private final String advertiseURI;
    267         private final Layout<? extends Serializable> layout;
    268
    269         /**
    270          * Constructor.
    271          *
    272          * @param append Append to existing file or truncate.
    273          * @param force forces the memory content to be written to the storage device on every event
    274          * @param regionLength length of the mapped region
    275          */
    276         public FactoryData(final boolean append, final boolean force, final int regionLength,
    277                 final String advertiseURI, final Layout<? extends Serializable> layout) {
    278             this.append = append;
    279             this.force = force;
    280             this.regionLength = regionLength;
    281             this.advertiseURI = advertiseURI;
    282             this.layout = layout;
    283         }
    284     }
    285
    286     /**
    287      * Factory to create a MemoryMappedFileManager.
    288      */
    289     private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
    290
    291         /**
    292          * Create a MemoryMappedFileManager.
    293          *
    294          * @param name The name of the File.
    295          * @param data The FactoryData
    296          * @return The MemoryMappedFileManager for the File.
    297          */
    298         @SuppressWarnings("resource")
    299         @Override
    300         public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
    301             final File file = new File(name);
    302             final File parent = file.getParentFile();
    303             if (null != parent && !parent.exists()) {
    304                 parent.mkdirs();
    305             }
    306             if (!data.append) {
    307                 file.delete();
    308             }
    309
    310             final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
    311             RandomAccessFile raf = null;
    312             try {
    313                 raf = new RandomAccessFile(name, "rw");
    314                 final long position = (data.append) ? raf.length() : 0;
    315                 raf.setLength(position + data.regionLength);
    316                 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
    317                         data.advertiseURI, data.layout);
    318             } catch (final Exception ex) {
    319                 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
    320                 Closer.closeSilently(raf);
    321             }
    322             return null;
    323         }
    324     }
    325 }