001    /*
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements. See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache license, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License. You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the license for the specific language governing permissions and
015     * limitations under the license.
016     */
017    package org.apache.logging.log4j.core.appender;
018    
019    import java.io.File;
020    import java.io.IOException;
021    import java.io.OutputStream;
022    import java.io.RandomAccessFile;
023    import java.io.Serializable;
024    import java.lang.reflect.Method;
025    import java.nio.ByteOrder;
026    import java.nio.MappedByteBuffer;
027    import java.nio.channels.FileChannel;
028    import java.security.AccessController;
029    import java.security.PrivilegedActionException;
030    import java.security.PrivilegedExceptionAction;
031    import java.util.HashMap;
032    import java.util.Map;
033    
034    import org.apache.logging.log4j.core.Layout;
035    import org.apache.logging.log4j.core.util.Assert;
036    import org.apache.logging.log4j.core.util.Closer;
037    import org.apache.logging.log4j.core.util.NullOutputStream;
038    
039    /**
040     * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
041     * memory and writes to this memory region.
042     * <p>
043     * 
044     * @see <a
045     *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
046     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
047     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
048     * @see <a
049     *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
050     * 
051     * @since 2.1
052     */
053    public class MemoryMappedFileManager extends OutputStreamManager {
054        static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
055        private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
056    
057        private final boolean isForce;
058        private final int regionLength;
059        private final String advertiseURI;
060        private final RandomAccessFile randomAccessFile;
061        private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
062        private MappedByteBuffer mappedBuffer;
063        private long mappingOffset;
064    
065        protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
066                final boolean force, final long position, final int regionLength, final String advertiseURI,
067                final Layout<? extends Serializable> layout) throws IOException {
068            super(os, fileName, layout);
069            this.isForce = force;
070            this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
071            this.regionLength = regionLength;
072            this.advertiseURI = advertiseURI;
073            this.isEndOfBatch.set(Boolean.FALSE);
074            this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
075            this.mappingOffset = position;
076        }
077    
078        /**
079         * Returns the MemoryMappedFileManager.
080         *
081         * @param fileName The name of the file to manage.
082         * @param append true if the file should be appended to, false if it should be overwritten.
083         * @param isForce true if the contents should be flushed to disk on every write
084         * @param regionLength The mapped region length.
085         * @param advertiseURI the URI to use when advertising the file
086         * @param layout The layout.
087         * @return A MemoryMappedFileManager for the File.
088         */
089        public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
090                final boolean isForce, final int regionLength, final String advertiseURI,
091                final Layout<? extends Serializable> layout) {
092            return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
093                    advertiseURI, layout), FACTORY);
094        }
095    
096        public Boolean isEndOfBatch() {
097            return isEndOfBatch.get();
098        }
099    
100        public void setEndOfBatch(final boolean isEndOfBatch) {
101            this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
102        }
103    
104        @Override
105        protected synchronized void write(final byte[] bytes, int offset, int length) {
106            super.write(bytes, offset, length); // writes to dummy output stream
107    
108            while (length > mappedBuffer.remaining()) {
109                final int chunk = mappedBuffer.remaining();
110                mappedBuffer.put(bytes, offset, chunk);
111                offset += chunk;
112                length -= chunk;
113                remap();
114            }
115            mappedBuffer.put(bytes, offset, length);
116    
117            // no need to call flush() if force is true,
118            // already done in AbstractOutputStreamAppender.append
119        }
120    
121        private synchronized void remap() {
122            final long offset = this.mappingOffset + mappedBuffer.position();
123            final int length = mappedBuffer.remaining() + regionLength;
124            try {
125                unsafeUnmap(mappedBuffer);
126                final long fileLength = randomAccessFile.length() + regionLength;
127                LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
128    
129                long startNanos = System.nanoTime();
130                randomAccessFile.setLength(fileLength);
131                final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
132                LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
133    
134                mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
135                mappingOffset = offset;
136            } catch (final Exception ex) {
137                LOGGER.error("Unable to remap " + getName() + ". " + ex);
138            }
139        }
140    
141        @Override
142        public synchronized void flush() {
143            mappedBuffer.force();
144        }
145    
146        @Override
147        public synchronized void close() {
148            final long position = mappedBuffer.position();
149            final long length = mappingOffset + position;
150            try {
151                unsafeUnmap(mappedBuffer);
152            } catch (final Exception ex) {
153                LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
154            }
155            try {
156                LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
157                        length, mappingOffset, position);
158                randomAccessFile.setLength(length);
159                randomAccessFile.close();
160            } catch (final IOException ex) {
161                LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
162            }
163        }
164    
165        public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
166                final int size) throws IOException {
167            for (int i = 1;; i++) {
168                try {
169                    LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
170    
171                    final long startNanos = System.nanoTime();
172                    final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
173                    map.order(ByteOrder.nativeOrder());
174    
175                    final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
176                    LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
177    
178                    return map;
179                } catch (final IOException e) {
180                    if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
181                        throw e;
182                    }
183                    LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
184                    if (i < 10) {
185                        Thread.yield();
186                    } else {
187                        try {
188                            Thread.sleep(1);
189                        } catch (final InterruptedException ignored) {
190                            Thread.currentThread().interrupt();
191                            throw e;
192                        }
193                    }
194                }
195            }
196        }
197    
198        private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
199            LOGGER.debug("MMapAppender unmapping old buffer...");
200            final long startNanos = System.nanoTime();
201            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
202                @Override
203                public Object run() throws Exception {
204                    final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
205                    getCleanerMethod.setAccessible(true);
206                    final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
207                    final Method cleanMethod = cleaner.getClass().getMethod("clean");
208                    cleanMethod.invoke(cleaner);
209                    return null;
210                }
211            });
212            final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
213            LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
214        }
215    
216        /**
217         * Returns the name of the File being managed.
218         *
219         * @return The name of the File being managed.
220         */
221        public String getFileName() {
222            return getName();
223        }
224    
225        /**
226         * Returns the length of the memory mapped region.
227         * 
228         * @return the length of the mapped region
229         */
230        public int getRegionLength() {
231            return regionLength;
232        }
233    
234        /**
235         * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
236         * {@code false} otherwise.
237         * 
238         * @return whether each write should be force-sync'ed
239         */
240        public boolean isImmediateFlush() {
241            return isForce;
242        }
243    
244        /**
245         * Gets this FileManager's content format specified by:
246         * <p>
247         * Key: "fileURI" Value: provided "advertiseURI" param.
248         * </p>
249         * 
250         * @return Map of content format keys supporting FileManager
251         */
252        @Override
253        public Map<String, String> getContentFormat() {
254            final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
255            result.put("fileURI", advertiseURI);
256            return result;
257        }
258    
259        /**
260         * Factory Data.
261         */
262        private static class FactoryData {
263            private final boolean append;
264            private final boolean force;
265            private final int regionLength;
266            private final String advertiseURI;
267            private final Layout<? extends Serializable> layout;
268    
269            /**
270             * Constructor.
271             *
272             * @param append Append to existing file or truncate.
273             * @param force forces the memory content to be written to the storage device on every event
274             * @param regionLength length of the mapped region
275             */
276            public FactoryData(final boolean append, final boolean force, final int regionLength,
277                    final String advertiseURI, final Layout<? extends Serializable> layout) {
278                this.append = append;
279                this.force = force;
280                this.regionLength = regionLength;
281                this.advertiseURI = advertiseURI;
282                this.layout = layout;
283            }
284        }
285    
286        /**
287         * Factory to create a MemoryMappedFileManager.
288         */
289        private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
290    
291            /**
292             * Create a MemoryMappedFileManager.
293             *
294             * @param name The name of the File.
295             * @param data The FactoryData
296             * @return The MemoryMappedFileManager for the File.
297             */
298            @SuppressWarnings("resource")
299            @Override
300            public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
301                final File file = new File(name);
302                final File parent = file.getParentFile();
303                if (null != parent && !parent.exists()) {
304                    parent.mkdirs();
305                }
306                if (!data.append) {
307                    file.delete();
308                }
309    
310                final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
311                RandomAccessFile raf = null;
312                try {
313                    raf = new RandomAccessFile(name, "rw");
314                    final long position = (data.append) ? raf.length() : 0;
315                    raf.setLength(position + data.regionLength);
316                    return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
317                            data.advertiseURI, data.layout);
318                } catch (final Exception ex) {
319                    LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
320                    Closer.closeSilently(raf);
321                }
322                return null;
323            }
324        }
325    }