    001    /*
    002     * Licensed to the Apache Software Foundation (ASF) under one or more
    003     * contributor license agreements. See the NOTICE file distributed with
    004     * this work for additional information regarding copyright ownership.
    005     * The ASF licenses this file to You under the Apache license, Version 2.0
    006     * (the "License"); you may not use this file except in compliance with
    007     * the License. You may obtain a copy of the License at
    008     *
    009     *      http://www.apache.org/licenses/LICENSE-2.0
    010     *
    011     * Unless required by applicable law or agreed to in writing, software
    012     * distributed under the License is distributed on an "AS IS" BASIS,
    013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014     * See the license for the specific language governing permissions and
    015     * limitations under the license.
    016     */
    017    package org.apache.logging.log4j.core.appender;
    018
    019    import java.io.File;
    020    import java.io.IOException;
    021    import java.io.OutputStream;
    022    import java.io.RandomAccessFile;
    023    import java.io.Serializable;
    024    import java.lang.reflect.Method;
    025    import java.nio.ByteOrder;
    026    import java.nio.MappedByteBuffer;
    027    import java.nio.channels.FileChannel;
    028    import java.security.AccessController;
    029    import java.security.PrivilegedActionException;
    030    import java.security.PrivilegedExceptionAction;
    031    import java.util.HashMap;
    032    import java.util.Map;
    033
    034    import org.apache.logging.log4j.core.Layout;
    035    import org.apache.logging.log4j.core.util.Assert;
    036    import org.apache.logging.log4j.core.util.Closer;
    037    import org.apache.logging.log4j.core.util.NullOutputStream;
    038
    039    /**
    040     * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
    041     * memory and writes to this memory region.
    042     * <p>
    043     *
    044     * @see <a
    045     *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
    046     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
    047     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
    048     * @see <a
    049     *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
    050     *
    051     * @since 2.1
    052     */
    053    public class MemoryMappedFileManager extends OutputStreamManager {
    054        static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
    055        private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
    056
    057        private final boolean isForce;
    058        private final int regionLength;
    059        private final String advertiseURI;
    060        private final RandomAccessFile randomAccessFile;
    061        private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
    062        private MappedByteBuffer mappedBuffer;
    063        private long mappingOffset;
    064
    065        protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
    066                final boolean force, final long position, final int regionLength, final String advertiseURI,
    067                final Layout<? extends Serializable> layout) throws IOException {
    068            super(os, fileName, layout);
    069            this.isForce = force;
    070            this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
    071            this.regionLength = regionLength;
    072            this.advertiseURI = advertiseURI;
    073            this.isEndOfBatch.set(Boolean.FALSE);
    074            this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
    075            this.mappingOffset = position;
    076        }
    077
    078        /**
    079         * Returns the MemoryMappedFileManager.
    080         *
    081         * @param fileName The name of the file to manage.
    082         * @param append true if the file should be appended to, false if it should be overwritten.
    083         * @param isForce true if the contents should be flushed to disk on every write
    084         * @param regionLength The mapped region length.
    085         * @param advertiseURI the URI to use when advertising the file
    086         * @param layout The layout.
    087         * @return A MemoryMappedFileManager for the File.
    088         */
    089        public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
    090                final boolean isForce, final int regionLength, final String advertiseURI,
    091                final Layout<? extends Serializable> layout) {
    092            return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
    093                    advertiseURI, layout), FACTORY);
    094        }
    095
    096        public Boolean isEndOfBatch() {
    097            return isEndOfBatch.get();
    098        }
    099
    100        public void setEndOfBatch(final boolean isEndOfBatch) {
    101            this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
    102        }
    103
    104        @Override
    105        protected synchronized void write(final byte[] bytes, int offset, int length) {
    106            super.write(bytes, offset, length); // writes to dummy output stream
    107
    108            while (length > mappedBuffer.remaining()) {
    109                final int chunk = mappedBuffer.remaining();
    110                mappedBuffer.put(bytes, offset, chunk);
    111                offset += chunk;
    112                length -= chunk;
    113                remap();
    114            }
    115            mappedBuffer.put(bytes, offset, length);
    116
    117            // no need to call flush() if force is true,
    118            // already done in AbstractOutputStreamAppender.append
    119        }
    120
    121        private synchronized void remap() {
    122            final long offset = this.mappingOffset + mappedBuffer.position();
    123            final int length = mappedBuffer.remaining() + regionLength;
    124            try {
    125                unsafeUnmap(mappedBuffer);
    126                final long fileLength = randomAccessFile.length() + regionLength;
    127                LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
    128
    129                long startNanos = System.nanoTime();
    130                randomAccessFile.setLength(fileLength);
    131                final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    132                LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
    133
    134                mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
    135                mappingOffset = offset;
    136            } catch (final Exception ex) {
    137                LOGGER.error("Unable to remap " + getName() + ". " + ex);
    138            }
    139        }
    140
    141        @Override
    142        public synchronized void flush() {
    143            mappedBuffer.force();
    144        }
    145
    146        @Override
    147        public synchronized void close() {
    148            final long position = mappedBuffer.position();
    149            final long length = mappingOffset + position;
    150            try {
    151                unsafeUnmap(mappedBuffer);
    152            } catch (final Exception ex) {
    153                LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
    154            }
    155            try {
    156                LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
    157                        length, mappingOffset, position);
    158                randomAccessFile.setLength(length);
    159                randomAccessFile.close();
    160            } catch (final IOException ex) {
    161                LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
    162            }
    163        }
    164
    165        public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
    166                final int size) throws IOException {
    167            for (int i = 1;; i++) {
    168                try {
    169                    LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
    170
    171                    final long startNanos = System.nanoTime();
    172                    final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
    173                    map.order(ByteOrder.nativeOrder());
    174
    175                    final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    176                    LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
    177
    178                    return map;
    179                } catch (final IOException e) {
    180                    if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
    181                        throw e;
    182                    }
    183                    LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
    184                    if (i < 10) {
    185                        Thread.yield();
    186                    } else {
    187                        try {
    188                            Thread.sleep(1);
    189                        } catch (final InterruptedException ignored) {
    190                            Thread.currentThread().interrupt();
    191                            throw e;
    192                        }
    193                    }
    194                }
    195            }
    196        }
    197
    198        private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
    199            LOGGER.debug("MMapAppender unmapping old buffer...");
    200            final long startNanos = System.nanoTime();
    201            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
    202                @Override
    203                public Object run() throws Exception {
    204                    final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
    205                    getCleanerMethod.setAccessible(true);
    206                    final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
    207                    final Method cleanMethod = cleaner.getClass().getMethod("clean");
    208                    cleanMethod.invoke(cleaner);
    209                    return null;
    210                }
    211            });
    212            final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
    213            LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
    214        }
    215
    216        /**
    217         * Returns the name of the File being managed.
    218         *
    219         * @return The name of the File being managed.
    220         */
    221        public String getFileName() {
    222            return getName();
    223        }
    224
    225        /**
    226         * Returns the length of the memory mapped region.
    227         *
    228         * @return the length of the mapped region
    229         */
    230        public int getRegionLength() {
    231            return regionLength;
    232        }
    233
    234        /**
    235         * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
    236         * {@code false} otherwise.
    237         *
    238         * @return whether each write should be force-sync'ed
    239         */
    240        public boolean isImmediateFlush() {
    241            return isForce;
    242        }
    243
    244        /**
    245         * Gets this FileManager's content format specified by:
    246         * <p>
    247         * Key: "fileURI" Value: provided "advertiseURI" param.
    248         * </p>
    249         *
    250         * @return Map of content format keys supporting FileManager
    251         */
    252        @Override
    253        public Map<String, String> getContentFormat() {
    254            final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
    255            result.put("fileURI", advertiseURI);
    256            return result;
    257        }
    258
    259        /**
    260         * Factory Data.
    261         */
    262        private static class FactoryData {
    263            private final boolean append;
    264            private final boolean force;
    265            private final int regionLength;
    266            private final String advertiseURI;
    267            private final Layout<? extends Serializable> layout;
    268
    269            /**
    270             * Constructor.
    271             *
    272             * @param append Append to existing file or truncate.
    273             * @param force forces the memory content to be written to the storage device on every event
    274             * @param regionLength length of the mapped region
    275             */
    276            public FactoryData(final boolean append, final boolean force, final int regionLength,
    277                    final String advertiseURI, final Layout<? extends Serializable> layout) {
    278                this.append = append;
    279                this.force = force;
    280                this.regionLength = regionLength;
    281                this.advertiseURI = advertiseURI;
    282                this.layout = layout;
    283            }
    284        }
    285
    286        /**
    287         * Factory to create a MemoryMappedFileManager.
    288         */
    289        private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
    290
    291            /**
    292             * Create a MemoryMappedFileManager.
    293             *
    294             * @param name The name of the File.
    295             * @param data The FactoryData
    296             * @return The MemoryMappedFileManager for the File.
    297             */
    298            @SuppressWarnings("resource")
    299            @Override
    300            public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
    301                final File file = new File(name);
    302                final File parent = file.getParentFile();
    303                if (null != parent && !parent.exists()) {
    304                    parent.mkdirs();
    305                }
    306                if (!data.append) {
    307                    file.delete();
    308                }
    309
    310                final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
    311                RandomAccessFile raf = null;
    312                try {
    313                    raf = new RandomAccessFile(name, "rw");
    314                    final long position = (data.append) ? raf.length() : 0;
    315                    raf.setLength(position + data.regionLength);
    316                    return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
    317                            data.advertiseURI, data.layout);
    318                } catch (final Exception ex) {
    319                    LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
    320                    Closer.closeSilently(raf);
    321                }
    322                return null;
    323            }
    324        }
    325    }