View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements.  See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache License, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License.  You may obtain a copy of the License at
    8    *
    9    *  http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   */
    17
    18  package org.apache.any23.plugin;
    19
    20  import org.apache.any23.cli.Tool;
    21  import org.apache.any23.configuration.DefaultConfiguration;
    22  import org.apache.any23.extractor.ExtractorFactory;
    23  import org.apache.any23.extractor.ExtractorGroup;
    24  import org.apache.any23.extractor.ExtractorRegistry;
    25  import org.slf4j.Logger;
    26  import org.slf4j.LoggerFactory;
    27
    28  import java.io.File;
    29  import java.io.FilenameFilter;
    30  import java.io.IOException;
    31  import java.net.MalformedURLException;
    32  import java.net.URL;
    33  import java.net.URLClassLoader;
    34  import java.util.ArrayList;
    35  import java.util.HashSet;
    36  import java.util.Iterator;
    37  import java.util.List;
    38  import java.util.ServiceLoader;
    39  import java.util.Set;
    40
    41  /**
    42   * The <i>Any23PluginManager</i> is responsible for inspecting dynamically the classpath and retrieving useful classes.
    43   *
    44   * @author Michele Mostarda (mostarda@fbk.eu)
    45   */
    46  public class Any23PluginManager {
    47
    48      /**
    49       * Any23 Command Line Interface package.
    50       */
    51      public static final String CLI_PACKAGE = Tool.class.getPackage().getName();
    52
    53      /**
    54       * Property where look for plugins.
    55       */
    56      public static final String PLUGIN_DIRS_PROPERTY = "any23.plugin.dirs";
    57
    58      /**
    59       * List separator for the string declaring the plugin list.
    60       */
    61      public static final String PLUGIN_DIRS_LIST_SEPARATOR = ":";
    62
    63      /**
    64       * Internal logger.
    65       */
    66      private static final Logger logger = LoggerFactory.getLogger(Any23PluginManager.class);
    67
    68      /**
    69       * Singleton lazy instance.
    70       */
    71      private static final Any23PluginManager instance = new Any23PluginManager();
    72
    73      /**
    74       * Internal class loader used to dynamically load classes.
    75       */
    76      private final DynamicClassLoader dynamicClassLoader;
    77
    78      /**
    79       * Constructor.
    80       */
    81      private Any23PluginManager() {
    82          dynamicClassLoader = new DynamicClassLoader();
    83      }
    84
    85      /**
    86       * @return a singleton instance of {@link Any23PluginManager}.
    87       */
    88      public static synchronized Any23PluginManager getInstance() {
    89          return instance;
    90      }
    91
    92      /**
    93       * Loads a <i>JAR</i> file in the classpath.
    94       *
    95       * @param jar
    96       *            the JAR file to be loaded.
    97       *
    98       * @return <code>true</code> if the JAR is added for the first time to the classpath, <code>false</code> otherwise.
    99       */
    100     public synchronized boolean loadJAR(File jar) {
    101         if (jar == null) {
    102             throw new NullPointerException("jar file cannot be null.");
    103         }
    104         if (!jar.isFile() && !jar.exists()) {
    105             throw new IllegalArgumentException(String.format(java.util.Locale.ROOT,
    106                     "Invalid JAR [%s], must be an existing file.", jar.getAbsolutePath()));
    107         }
    108         return dynamicClassLoader.addJAR(jar);
    109     }
    110
    111     /**
    112      * Loads a list of <i>JAR</i>s in the classpath.
    113      *
    114      * @param jars
    115      *            list of JARs to be loaded.
    116      *
    117      * @return list of exceptions raised during the loading.
    118      */
    119     public synchronized Throwable[] loadJARs(File... jars) {
    120         final List<Throwable> result = new ArrayList<>();
    121         for (File jar : jars) {
    122             try {
    123                 loadJAR(jar);
    124             } catch (Throwable t) {
    125                 result.add(new IllegalArgumentException(
    126                         String.format(java.util.Locale.ROOT, "Error while loading jar [%s]", jar.getAbsolutePath()),
    127                         t));
    128             }
    129         }
    130         return result.toArray(new Throwable[result.size()]);
    131     }
    132
    133     /**
    134      * Loads a <i>classes</i> directory in the classpath.
    135      *
    136      * @param classDir
    137      *            the directory to be loaded.
    138      *
    139      * @return <code>true</code> if the directory is added for the first time to the classpath, <code>false</code>
    140      *         otherwise.
    141      */
    142     public synchronized boolean loadClassDir(File classDir) {
    143         if (classDir == null) {
    144             throw new NullPointerException("classDir cannot be null.");
    145         }
    146         if (!classDir.isDirectory() && !classDir.exists()) {
    147             throw new IllegalArgumentException(String.format(java.util.Locale.ROOT,
    148                     "Invalid class dir [%s], must be an existing file.", classDir.getAbsolutePath()));
    149         }
    150         return dynamicClassLoader.addClassDir(classDir);
    151     }
    152
    153     /**
    154      * Loads a list of class dirs in the classpath.
    155      *
    156      * @param classDirs
    157      *            list of class dirs to be loaded.
    158      *
    159      * @return list of exceptions raised during the loading.
    160      */
    161     public synchronized Throwable[] loadClassDirs(File... classDirs) {
    162         final List<Throwable> result = new ArrayList<>();
    163         for (File classDir : classDirs) {
    164             try {
    165                 loadClassDir(classDir);
    166             } catch (Throwable t) {
    167                 result.add(new IllegalArgumentException(String.format(java.util.Locale.ROOT,
    168                         "Error while loading class dir [%s]", classDir.getAbsolutePath()), t));
    169             }
    170         }
    171         return result.toArray(new Throwable[result.size()]);
    172     }
    173
    174     /**
    175      * Loads all the JARs detected in a given directory.
    176      *
    177      * @param jarDir
    178      *            directory containing the JARs to be loaded. Example
    179      *            '/usr/local/apache-tomcat-7.0.72/webapps/apache-any23-service-2.2-SNAPSHOT/WEB-INF/lib/apache-any23-openie'
    180      *
    181      * @return <code>true</code> if all JARs in dir are loaded.
    182      */
    183     public synchronized boolean loadJARDir(File jarDir) {
    184         if (jarDir == null)
    185             throw new NullPointerException("JAR dir must be not null.");
    186         if (!jarDir.exists())
    187             throw new IllegalArgumentException("Given directory doesn't exist:" + jarDir.getAbsolutePath());
    188         if (!jarDir.isDirectory())
    189             throw new IllegalArgumentException(
    190                     "given file exists and it is not a directory: " + jarDir.getAbsolutePath());
    191         boolean loaded = true;
    192         for (File jarFile : jarDir.listFiles(new FilenameFilter() {
    193             @Override
    194             public boolean accept(File dir, String name) {
    195                 return name.endsWith(".jar");
    196             }
    197         })) {
    198             loaded &= loadJAR(jarFile);
    199         }
    200         return loaded;
    201     }
    202
    203     /**
    204      * Loads a generic list of files, trying to determine the type of every file.
    205      *
    206      * @param files
    207      *            list of files to be loaded.
    208      *
    209      * @return list of errors occurred during loading.
    210      */
    211     public synchronized Throwable[] loadFiles(File... files) {
    212         final List<Throwable> errors = new ArrayList<>();
    213         for (File file : files) {
    214             try {
    215                 if (file.isFile() && file.getName().endsWith(".jar")) {
    216                     loadJAR(file);
    217                 } else if (file.isDirectory()) {
    218                     if (file.getName().endsWith("classes")) {
    219                         loadClassDir(file);
    220                     } else {
    221                         loadJARDir(file);
    222                     }
    223                 } else {
    224                     throw new IllegalArgumentException("Cannot handle file " + file.getAbsolutePath());
    225                 }
    226             } catch (Throwable t) {
    227                 errors.add(t);
    228             }
    229         }
    230         return errors.toArray(new Throwable[errors.size()]);
    231     }
    232
    233     /**
    234      * Returns all classes within the specified <code>packageName</code> satisfying the given class <code>filter</code>.
    235      * The search is performed on the static classpath (the one the application started with) and the dynamic classpath
    236      * (the one specified using the load methods).
    237      *
    238      * @param <T>
    239      *            type of filtered class.
    240      * @param type
    241      *            of filtered class.
    242      *
    243      * @return list of matching classes.
    244      *
    245      * @throws IOException
    246      *             if there is an error obtaining plugins.
    247      */
    248     public synchronized <T> Iterator<T> getPlugins(final Class<T> type) throws IOException {
    249         return ServiceLoader.load(type, dynamicClassLoader).iterator();
    250     }
    251
    252     /**
    253      * Returns the list of all the {@link Tool} classes declared within the classpath.
    254      *
    255      * @return not <code>null</code> list of tool classes.
    256      *
    257      * @throws IOException
    258      *             if there is an error obtaining {@link org.apache.any23.cli.Tool}'s from the classpath.
    259      */
    260     public synchronized Iterator<Tool> getTools() throws IOException {
    261         return getPlugins(Tool.class);
    262     }
    263
    264     /**
    265      * List of {@link ExtractorPlugin} classes declared within the classpath.
    266      *
    267      * @return not <code>null</code> list of plugin classes.
    268      *
    269      * @throws IOException
    270      *             if there is an error obtaining Extractors.
    271      */
    272     @SuppressWarnings("rawtypes")
    273     public synchronized Iterator<ExtractorFactory> getExtractors() throws IOException {
    274         return getPlugins(ExtractorFactory.class);
    275     }
    276
    277     /**
    278      * Loads plugins from a list of specified locations.
    279      *
    280      * @param pluginLocations
    281      *            list of locations.
    282      *
    283      * @return a report about the loaded plugins.
    284      */
    285     public synchronized String loadPlugins(File... pluginLocations) {
    286         final StringBuilder report = new StringBuilder();
    287         report.append("\nLoading plugins from locations {\n");
    288         for (File pluginLocation : pluginLocations) {
    289             report.append(pluginLocation.getAbsolutePath()).append('\n');
    290         }
    291         report.append("}\n");
    292
    293         final Throwable[] errors = loadFiles(pluginLocations);
    294         if (errors.length > 0) {
    295             report.append("The following errors occurred while loading plugins {\n");
    296             for (Throwable error : errors) {
    297                 report.append(error);
    298                 report.append("\n\n\n");
    299             }
    300             report.append("}\n");
    301         }
    302         return report.toString();
    303     }
    304
    305     /**
    306      * Configures a new list of extractors containing the extractors declared in <code>initialExtractorGroup</code> and
    307      * also the extractors detected in classpath specified by <code>pluginLocations</code>.
    308      *
    309      * @param pluginLocations
    310      *            path locations of plugins.
    311      *
    312      * @return full list of extractors.
    313      *
    314      * @throws java.io.IOException
    315      *             if there is an error locating the plugin(s).
    316      * @throws IllegalAccessException
    317      *             if there are access permissions for plugin(s).
    318      * @throws InstantiationException
    319      *             if there is an error instantiating plugin(s).
    320      */
    321     public synchronized ExtractorGroup configureExtractors(final File... pluginLocations)
    322             throws IOException, IllegalAccessException, InstantiationException {
    323
    324         final String pluginsReport = loadPlugins(pluginLocations);
    325         logger.info(pluginsReport);
    326
    327         final StringBuilder report = new StringBuilder();
    328         try {
    329             final List<ExtractorFactory<?>> newFactoryList = new ArrayList<>();
    330             @SuppressWarnings("rawtypes")
    331             Iterator<ExtractorFactory> extractors = getExtractors();
    332             while (extractors.hasNext()) {
    333                 ExtractorFactory<?> factory = extractors.next();
    334
    335                 report.append("\n - found plugin: ").append(factory.getExtractorName()).append("\n");
    336
    337                 newFactoryList.add(factory);
    338             }
    339
    340             if (newFactoryList.isEmpty()) {
    341                 report.append("\n=== No plugins have been found.===\n");
    342             }
    343
    344             return new ExtractorGroup(newFactoryList);
    345         } finally {
    346             logger.info(report.toString());
    347         }
    348     }
    349
    350     /**
    351      * Configures a new list of extractors containing the extractors declared in <code>initialExtractorGroup</code> and
    352      * also the extractors detected in classpath specified by the default configuration.
    353      *
    354      * @param initialExtractorGroup
    355      *            initial list of extractors.
    356      *
    357      * @return full list of extractors.
    358      *
    359      * @throws java.io.IOException
    360      *             if there is an error locating the extractor(s).
    361      * @throws IllegalAccessException
    362      *             if there are access permissions for extractor(s).
    363      * @throws InstantiationException
    364      *             if there is an error instantiating extractor(s).
    365      */
    366     public synchronized ExtractorGroup configureExtractors(ExtractorGroup initialExtractorGroup)
    367             throws IOException, InstantiationException, IllegalAccessException {
    368         final String pluginDirs = DefaultConfiguration.singleton().getPropertyOrFail(PLUGIN_DIRS_PROPERTY);
    369         final File[] pluginLocations = getPluginLocations(pluginDirs);
    370         return configureExtractors(pluginLocations);
    371     }
    372
    373     /**
    374      * Returns an extractor group containing both the default extractors declared by the
    375      * {@link org.apache.any23.extractor.ExtractorRegistry} and the {@link ExtractorPlugin}s.
    376      *
    377      * @param registry
    378      *            an {@link org.apache.any23.extractor.ExtractorRegistry}
    379      * @param pluginLocations
    380      *            optional list of plugin locations.
    381      *
    382      * @return a not <code>null</code> and not empty extractor group.
    383      *
    384      * @throws java.io.IOException
    385      *             if there is an error locating the extractor group.
    386      * @throws IllegalAccessException
    387      *             if there are access permissions for the extractor group.
    388      * @throws InstantiationException
    389      *             if there is an error instantiating the extractor group.
    390      */
    391     public synchronized ExtractorGroup getApplicableExtractors(ExtractorRegistry registry, File... pluginLocations)
    392             throws IOException, IllegalAccessException, InstantiationException {
    393         return configureExtractors(pluginLocations);
    394     }
    395
    396     /**
    397      * Returns an {@link Iterator} of tools that have been detected within the given list of locations.
    398      *
    399      * @param pluginLocations
    400      *            list of plugin locations.
    401      *
    402      * @return set of detected tools.
    403      *
    404      * @throws IOException
    405      *             if there is an error acessing {@link org.apache.any23.cli.Tool}'s.
    406      */
    407     public synchronized Iterator<Tool> getApplicableTools(File... pluginLocations) throws IOException {
    408         final String report = loadPlugins(pluginLocations);
    409         logger.info(report);
    410         return getTools();
    411     }
    412
    413     /**
    414      * Converts a column separated list of dirs in a list of files.
    415      *
    416      * @param pluginDirsList
    417      *
    418      * @return
    419      */
    420     private File[] getPluginLocations(String pluginDirsList) {
    421         final String[] locationsStr = pluginDirsList.split(PLUGIN_DIRS_LIST_SEPARATOR);
    422         final List<File> locations = new ArrayList<>();
    423         for (String locationStr : locationsStr) {
    424             final File location = new File(locationStr);
    425             if (!location.exists()) {
    426                 throw new IllegalArgumentException(
    427                         String.format(java.util.Locale.ROOT, "Plugin location '%s' cannot be found.", locationStr));
    428             }
    429             locations.add(location);
    430         }
    431         return locations.toArray(new File[locations.size()]);
    432     }
    433
    434     /**
    435      * Dynamic local file class loader.
    436      */
    437     private static final class DynamicClassLoader extends URLClassLoader {
    438
    439         private final Set<String> addedURLs = new HashSet<>();
    440
    441         private final List<File> jars;
    442
    443         private final List<File> dirs;
    444
    445         public DynamicClassLoader(URL[] urls) {
    446             super(urls, Any23PluginManager.class.getClassLoader());
    447             jars = new ArrayList<>();
    448             dirs = new ArrayList<>();
    449         }
    450
    451         public DynamicClassLoader() {
    452             this(new URL[0]);
    453         }
    454
    455         public boolean addClassDir(File classDir) {
    456             final String urlPath = "file://" + classDir.getAbsolutePath() + "/";
    457             try {
    458                 if (addURL(urlPath)) {
    459                     dirs.add(classDir);
    460                     return true;
    461                 }
    462                 return false;
    463             } catch (MalformedURLException murle) {
    464                 throw new RuntimeException("Invalid dir URL.", murle);
    465             }
    466         }
    467
    468         public boolean addJAR(File jar) {
    469             final String urlPath = "jar:file://" + jar.getAbsolutePath() + "!/";
    470             try {
    471                 if (addURL(urlPath)) {
    472                     jars.add(jar);
    473                     return true;
    474                 }
    475                 return false;
    476             } catch (MalformedURLException murle) {
    477                 throw new RuntimeException("Invalid JAR URL.", murle);
    478             }
    479         }
    480
    481         private boolean addURL(String urlPath) throws MalformedURLException {
    482             if (addedURLs.contains(urlPath)) {
    483                 return false;
    484             }
    485             super.addURL(new URL(urlPath));
    486             addedURLs.add(urlPath);
    487             return true;
    488         }
    489     }
    490
    491 }