/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.netbeans.nbbuild;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.BuildListener;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.taskdefs.Property;
import org.apache.tools.ant.types.Path;
import org.apache.tools.ant.util.FileUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

/**
 * Scans for known modules.
 * Precise algorithm summarized in issue #42681 and issue #58966.
 * @author Jesse Glick
 */
final class ModuleListParser {

    private static Map<File,Map<String,Entry>> SOURCE_SCAN_CACHE = new HashMap<File,Map<String,Entry>>();
    private static Map<File,Map<String,Entry>> SUITE_SCAN_CACHE = new HashMap<File,Map<String,Entry>>();
    private static Map<File,Entry> STANDALONE_SCAN_CACHE = new HashMap<File,Entry>();
    private static Map<File,Map<String,Entry>> BINARY_SCAN_CACHE = new HashMap<File,Map<String,Entry>>();
    
    /** Clear caches. Cf. #71130. */
    public static void resetCaches() {
        SOURCE_SCAN_CACHE.clear();
        SUITE_SCAN_CACHE.clear();
        STANDALONE_SCAN_CACHE.clear();
        BINARY_SCAN_CACHE.clear();
    }
    
    /** Synch with org.netbeans.modules.apisupport.project.universe.ModuleList.FOREST: */
    private static final String[] FOREST = {
        /*root*/null,
        "contrib",
        "otherlicenses",
        // do not scan in misc; any real modules would have been put in contrib
        // Will there be other subtrees in the future (not using suites)?
    };
    /**
     * Find all NBM projects in a root, possibly from cache.
     */
    private static Map<String,Entry> scanNetBeansOrgSources(File root, Map<String,Object> properties, Project project) throws IOException {
        Map<String,Entry> entries = SOURCE_SCAN_CACHE.get(root);
        if (entries == null) {
            // Similar to #62221: if just invoked from a module in standard clusters, only scan those clusters (faster):
            Set<String> standardModules = new HashSet<>();
            boolean doFastScan = false;
            String basedir = (String) properties.get("basedir");
            String clusterList = (String) properties.get("nb.clusters.list");
            if (basedir != null) {
                File basedirF = new File(basedir);
                if (clusterList == null) {
                    String config = (String) properties.get("cluster.config");
                    if (config != null) {
                        clusterList = (String) properties.get("clusters.config." + config + ".list");
                    }
                }
                if (clusterList != null) {
                    StringTokenizer tok = new StringTokenizer(clusterList, ", ");
                    while (tok.hasMoreTokens()) {
                        String clusterName = tok.nextToken();
                        String moduleList = (String) properties.get(clusterName);
                        if (moduleList != null) {
                            // Hack to treat libs.junit4 as if it were in platform for purposes of building, yet build to another cluster.
                            if (clusterName.equals("nb.cluster.platform")) {
                                moduleList += ",libs.junit4";
                            } else if (clusterName.equals("nb.cluster.stableuc")) {
                                moduleList = moduleList.replace(",libs.junit4", "");
                            }


                            StringTokenizer tok2 = new StringTokenizer(moduleList, ", ");
                            while (tok2.hasMoreTokens()) {
                                String module = tok2.nextToken();
                                standardModules.add(module);
                                // doFastScan |= new File(root, module.replace('/', File.separatorChar)).equals(basedirF);
                            }
                        }
                    }
                }
            }
            String p = (String) properties.get("netbeans.dest.dir"); // NOI18N
            int hash = root.hashCode() * 7 + (p == null ? 1 : p.hashCode());
            File scanCache = new File(System.getProperty("java.io.tmpdir"), "nb-scan-cache-" + String.format("%x", hash) + "-" + (doFastScan ? "standard" : "full") + ".ser");
            if (scanCache.isFile()) {
                if (project != null) {
                    project.log("Loading module list from " + scanCache);
                }
                try {
                    try (InputStream is = new FileInputStream(scanCache)) {
                        ObjectInput oi = new ObjectInputStream(new BufferedInputStream(is));
                        @SuppressWarnings("unchecked") Map<File,Long[]> timestampsAndSizes = (Map) oi.readObject();
                        boolean matches = true;
                        for (Map.Entry<File,Long[]> entry : timestampsAndSizes.entrySet()) {
                            File f = entry.getKey();
                            if (f.lastModified() != entry.getValue()[0] || f.length() != entry.getValue()[1]) {
                                if (project != null) {
                                    project.log("Cache ignored due to modifications in " + f);
                                }
                                matches = false;
                                break;
                            }
                        }
                        if (doFastScan) {
                            @SuppressWarnings("unchecked") Set<String> storedStandardModules = (Set) oi.readObject();
                            if (!standardModules.equals(storedStandardModules)) {
                                Set<String> added = new TreeSet<>(standardModules);
                                added.removeAll(storedStandardModules);
                                Set<String> removed = new TreeSet<>(storedStandardModules);
                                removed.removeAll(standardModules);
                                project.log("Cache ignored due to changes in modules among standard clusters: + " + added + " - " + removed);
                                matches = false;
                            }
                        }
                        File myProjectXml = project.resolveFile("nbproject/project.xml");
                        if (myProjectXml.isFile() && !timestampsAndSizes.containsKey(myProjectXml)) {
                            project.log("Cache ignored since it has no mention of " + myProjectXml);
                            matches = false; // #118098
                        }
                        if (matches) {
                            @SuppressWarnings("unchecked") Map<String,Entry> _entries = (Map) oi.readObject();
                            entries = _entries;
                            if (project != null) {
                                project.log("Loaded modules: " + entries.keySet(), Project.MSG_DEBUG);
                            }
                        }
                    }
                } catch (Exception x) {
                    if (project != null) {
                        project.log("Error loading " + scanCache + ": " + x, Project.MSG_WARN);
                    }
                }
            }
            if (entries == null) {
                entries = new HashMap<>();
                Map<File,Long[]> timestampsAndSizes = new HashMap<>();
                registerTimestampAndSize(new File(root, "nbbuild" + File.separatorChar + "cluster.properties"), timestampsAndSizes);
                registerTimestampAndSize(new File(root, "nbbuild" + File.separatorChar + "build.properties"), timestampsAndSizes);
                registerTimestampAndSize(new File(root, "nbbuild" + File.separatorChar + "user.build.properties"), timestampsAndSizes);

                doFastScan = false;

                if (doFastScan) {
                    if (project != null) {
                        project.log("Scanning for modules in " + root + " among standard clusters");
                    }
                    for (String module : standardModules) {
                        scanPossibleProject(new File(root, module.replace('/', File.separatorChar)), entries, properties, module, ModuleType.NB_ORG, project, timestampsAndSizes);
                    }
                } else {
                    // Might be an extra module (e.g. something in contrib); need to scan everything.
                    if (project != null) {
                        project.log("Scanning for modules in " + root);
                        project.log("Quick scan mode disabled since " + basedir + " not among standard modules of " + root + " which are " + standardModules, Project.MSG_VERBOSE);
                    }
                    List<String> subDirs = new ArrayList<>();
                    subDirs.addAll(Arrays.asList(FOREST));
                    String allClusters = (String) properties.get("clusters.config.full.list");
                    if(allClusters == null) {
                        allClusters = "";
                    }
                    StringTokenizer tok = new StringTokenizer(allClusters, ", ");
                    while (tok.hasMoreTokens()) {
                        String clusterName = tok.nextToken();
                        String clusterDir = (String) properties.get(clusterName + ".dir");
                        if (subDirs.contains(clusterDir)) {
                            continue;
                        }
                        subDirs.add(clusterDir);
                    }
                    for (String tree : subDirs) {
                        File dir = tree == null ? root : new File(root, tree);
                        File[] kids = dir.listFiles();
                        if (kids == null) {
                            continue;
                        }
                        for (File kid : kids) {
                            if (!kid.isDirectory()) {
                                continue;
                            }
                            String name = kid.getName();
                            String path = tree == null ? name : tree + "/" + name;
                            scanPossibleProject(kid, entries, properties, path, ModuleType.NB_ORG, project, timestampsAndSizes);
                        }
                        
                    }
                }
                if (project != null) {
                    project.log("Found modules: " + entries.keySet(), Project.MSG_VERBOSE);
                    project.log("Cache depends on files: " + timestampsAndSizes.keySet(), Project.MSG_DEBUG);
                }
                scanCache.getParentFile().mkdirs();
                try (OutputStream os = new FileOutputStream(scanCache)) {
                    ObjectOutput oo = new ObjectOutputStream(os);
                    oo.writeObject(timestampsAndSizes);
                    if (doFastScan) {
                        oo.writeObject(standardModules);
                    }
                    oo.writeObject(entries);
                    oo.flush();
                }
            }
            SOURCE_SCAN_CACHE.put(root, entries);
        }
        return entries;
    }

    private static void registerTimestampAndSize(File f, Map<File,Long[]> timestampsAndSizes) {
        if (timestampsAndSizes != null) {
            timestampsAndSizes.put(f, new Long[] {f.lastModified(), f.length()});
        }
    }
    
    /**
     * Check a single dir to see if it is an NBM project, and if so, register it.
     */
    private static boolean scanPossibleProject(File dir, Map<String,Entry> entries, Map<String,Object> properties,
            String path, ModuleType moduleType, Project project, Map<File,Long[]> timestampsAndSizes) throws IOException {
        File nbproject = new File(dir, "nbproject");
        File projectxml = new File(nbproject, "project.xml");
        if (!projectxml.isFile()) {
            return false;
        }
        registerTimestampAndSize(projectxml, timestampsAndSizes);
        Document doc;
        try {
            doc = XMLUtil.parse(new InputSource(projectxml.toURI().toString()),
                                     false, true, /*XXX*/null, null);
        } catch (Exception e) { // SAXException, IOException (#60295: e.g. encoding problem in XML)
            // Include \n so that following line can be hyperlinked
            throw (IOException) new IOException("Error parsing project file\n" + projectxml + ": " + e.getMessage()).initCause(e);
        }
        Element typeEl = XMLUtil.findElement(doc.getDocumentElement(), "type", ParseProjectXml.PROJECT_NS);
        if (!XMLUtil.findText(typeEl).equals("org.netbeans.modules.apisupport.project")) {
            return false;
        }
        Element configEl = XMLUtil.findElement(doc.getDocumentElement(), "configuration", ParseProjectXml.PROJECT_NS);
        Element dataEl = ParseProjectXml.findNBMElement(configEl, "data");
        if (dataEl == null) {
            if (project != null) {
                project.log(projectxml.toString() + ": warning: module claims to be a NBM project but is missing <data xmlns=\"" + ParseProjectXml.NBM_NS3 + "\">; maybe an old NB 4.[01] project?", Project.MSG_WARN);
            }
            return false;
        }
        Element cnbEl = ParseProjectXml.findNBMElement(dataEl, "code-name-base");
        String cnb = XMLUtil.findText(cnbEl);
        if (moduleType == ModuleType.NB_ORG && project != null) {
            String expectedDirName = abbreviate(cnb);
            String actualDirName = dir.getName();
            if (!actualDirName.equals(expectedDirName)) {
                throw new IOException("Expected module to be in dir named " + expectedDirName + " but was actually found in dir named " + actualDirName);
            }
        }
        // Clumsy but the best way I know of to evaluate properties.
        Project fakeproj = new Project();
        if (project != null) {
            // Try to debug any problems in the following definitions (cf. #59849).
            for(BuildListener bl: project.getBuildListeners()) {
                fakeproj.addBuildListener(bl);
            }
        }
        fakeproj.setBaseDir(dir); // in case ${basedir} is used somewhere
        Property faketask = new Property();
        faketask.setProject(fakeproj);
        switch (moduleType) {
        case NB_ORG:
            // do nothing here
            break;
        case SUITE:
            faketask.setFile(new File(nbproject, "private/suite-private.properties"));
            faketask.execute();
            faketask.setFile(new File(nbproject, "suite.properties"));
            faketask.execute();
            faketask.setFile(new File(fakeproj.replaceProperties("${suite.dir}/nbproject/private/platform-private.properties")));
            faketask.execute();
            faketask.setFile(new File(fakeproj.replaceProperties("${suite.dir}/nbproject/platform.properties")));
            faketask.execute();
            break;
        case STANDALONE:
            faketask.setFile(new File(nbproject, "private/platform-private.properties"));
            faketask.execute();
            faketask.setFile(new File(nbproject, "platform.properties"));
            faketask.execute();
            break;
        default:
            assert false : moduleType;
        }
        File privateProperties = new File(nbproject, "private/private.properties".replace('/', File.separatorChar));
        registerTimestampAndSize(privateProperties, timestampsAndSizes);
        faketask.setFile(privateProperties);
        faketask.execute();
        File projectProperties = new File(nbproject, "project.properties");
        registerTimestampAndSize(projectProperties, timestampsAndSizes);
        faketask.setFile(projectProperties);
        faketask.execute();
        faketask.setFile(null);
        faketask.setName("module.jar.dir");
        faketask.setValue("modules");
        faketask.execute();
        assert fakeproj.getProperty("module.jar.dir") != null : fakeproj.getProperties();
        faketask.setName("module.jar.basename");
        faketask.setValue(cnb.replace('.', '-') + ".jar");
        faketask.execute();
        faketask.setName("module.jar");
        faketask.setValue(fakeproj.replaceProperties("${module.jar.dir}/${module.jar.basename}"));
        faketask.execute();
        String id = null;
        switch (moduleType) {
        case NB_ORG:
            assert path != null;
            // Find the associated cluster.
            // first try direct mapping in nbbuild/netbeans/moduleCluster.properties
            String[] clusterDir = { (String) properties.get(path + ".dir") };
            if (clusterDir[0] != null) {
                final String subStr = clusterDir[0].substring(clusterDir[0].lastIndexOf('/') + 1);
                if (path.startsWith(subStr + "/")) {
                    id = path.substring(subStr.length() + 1);
                }
                clusterDir[0] = subStr;
            } else {
                id = SetCluster.findClusterAndId("cluster.dir", clusterDir, properties, path, faketask, "extra");
            }
            faketask.setName("cluster.dir");
            faketask.setValue(clusterDir[0]);
            faketask.execute();
            faketask.setName("netbeans.dest.dir");
            faketask.setValue(properties.get("netbeans.dest.dir"));
            faketask.execute();
            faketask.setName("cluster");
            faketask.setValue(fakeproj.replaceProperties("${netbeans.dest.dir}/${cluster.dir}"));
            faketask.execute();
            break;
        case SUITE:
            assert path == null;
            faketask.setName("suite.dir");
            faketask.setValue(properties.get("suite.dir"));
            faketask.execute();
            faketask.setName("suite.build.dir");
            faketask.setValue(fakeproj.replaceProperties("${suite.dir}/build"));
            faketask.execute();
            faketask.setName("cluster");
            faketask.setValue(fakeproj.replaceProperties("${suite.build.dir}/cluster"));
            faketask.execute();
            break;
        case STANDALONE:
            assert path == null;
            faketask.setName("build.dir");
            faketask.setValue(fakeproj.replaceProperties("${basedir}/build"));
            faketask.execute();
            faketask.setName("cluster");
            faketask.setValue(fakeproj.replaceProperties("${build.dir}/cluster"));
            faketask.execute();
            break;
        default:
            assert false : moduleType;
        }
        File jar = fakeproj.resolveFile(fakeproj.replaceProperties("${cluster}/${module.jar}"));
        List<File> exts = new ArrayList<>();
        for (Element ext : XMLUtil.findSubElements(dataEl)) {
            if (!ext.getLocalName().equals("class-path-extension")) {
                continue;
            }
            Element binaryOrigin = ParseProjectXml.findNBMElement(ext, "binary-origin");
            File origBin = null;
            if (binaryOrigin != null) {
                String reltext = XMLUtil.findText(binaryOrigin);
                String nball = (String) properties.get("nb_all");
                if (nball != null) {
                    faketask.setName("nb_all");
                    faketask.setValue(nball);
                    faketask.execute();
                }
                fakeproj.setBaseDir(dir);
                origBin = fakeproj.resolveFile(fakeproj.replaceProperties(reltext));
            } 

            File resultBin = null;
            if (origBin == null || !origBin.exists()) {
                Element runtimeRelativePath = ParseProjectXml.findNBMElement(ext, "runtime-relative-path");
                if (runtimeRelativePath == null) {
                    throw new IOException("Have malformed <class-path-extension> in " + projectxml);
                }
                String reltext = XMLUtil.findText(runtimeRelativePath);
                if (reltext != null) {
                    // No need to evaluate property refs in it - it is *not* substitutable-text in the schema.
                    String nball = (String) properties.get("nb_all");
                    resultBin = new File(jar.getParentFile(), reltext.replace('/', File.separatorChar));
                    resultBin = fixFxRtJar(resultBin, nball);
                }
            }

            if (origBin != null) {
                exts.add(origBin);
            }

            if (resultBin != null) {
                exts.add(resultBin);
            }
        }
        List<String> prereqs = new ArrayList<>();
        List<String> rundeps = new ArrayList<>();
        Element depsEl = ParseProjectXml.findNBMElement(dataEl, "module-dependencies");
        if (depsEl == null) {
            throw new IOException("Malformed project file " + projectxml);
        }
        Element testDepsEl = ParseProjectXml.findNBMElement(dataEl,"test-dependencies");
         //compileDeps = Collections.emptyList();
        Map<String,String[]> compileTestDeps = new HashMap<>();
        if (testDepsEl != null) {
            for (Element depssEl : XMLUtil.findSubElements(testDepsEl)) {
                String testtype = ParseProjectXml.findTextOrNull(depssEl,"name") ;
                if (testtype == null) {
                    throw new IOException("Must declare <name>unit</name> (e.g.) in <test-type> in " + projectxml);
                }
                List<String> compileDepsList = new ArrayList<>();
                for (Element dep : XMLUtil.findSubElements(depssEl)) {
                    if (dep.getTagName().equals("test-dependency")) {
                        if (ParseProjectXml.findNBMElement(dep,"test") != null)  {
                            compileDepsList.add(ParseProjectXml.findTextOrNull(dep, "code-name-base"));
                        } 
                    }
                }
                compileTestDeps.put(testtype, compileDepsList.toArray(new String[0]));
            }
        } 
        for (Element dep : XMLUtil.findSubElements(depsEl)) {
            Element cnbEl2 = ParseProjectXml.findNBMElement(dep, "code-name-base");
            if (cnbEl2 == null) {
                throw new IOException("Malformed project file " + projectxml);
            }
            String cnb2 = XMLUtil.findText(cnbEl2);
            rundeps.add(cnb2);
            if (ParseProjectXml.findNBMElement(dep, "build-prerequisite") == null) {
                continue;
            }
            prereqs.add(cnb2);
        }
        String cluster = fakeproj.getProperty("cluster.dir"); // may be null
        Entry entry = new Entry(cnb, jar, exts.toArray(new File[exts.size()]), dir, path, id == null ? path : id,
                prereqs.toArray(new String[prereqs.size()]), 
                cluster, 
                rundeps.toArray(new String[rundeps.size()]),
                compileTestDeps,
                new File(dir, "module-auto-deps.xml")
                );
        if (entries.containsKey(cnb)) {
            throw new IOException("Duplicated module " + cnb + ": found in " + entries.get(cnb) + " and " + entry);
        } else {
            entries.put(cnb, entry);
        }
        return true;
    }

    private static File fixFxRtJar(File resultBin, String nball) {
        final String path = resultBin.getPath().replace(File.separatorChar, '/');
        if (!resultBin.exists() && path.contains("${java.home}/lib/ext/jfxrt.jar")) {
            String jhm = System.getProperty("java.home");
            resultBin = new File(new File(new File(new File(jhm), "lib"), "ext"), "jfxrt.jar");
            if (!resultBin.exists()) {
                File jdk7 = new File(new File(new File(jhm), "lib"), "jfxrt.jar");
                if (jdk7.exists()) {
                    resultBin = jdk7;
                } else if (nball != null) {
                    File external = new File(new File(new File(new File(nball), "libs.javafx"), "external"), "jfxrt.jar");
                    if (external.exists()) {
                        resultBin = external;
                    }
                }
            }
        }
        return resultBin;
    }
    static String abbreviate(String cnb) {
        return cnb.replaceFirst("^org\\.netbeans\\.modules\\.", ""). // NOI18N
                   replaceFirst("^org\\.netbeans\\.(libs|lib|api|spi|core)\\.", "$1."). // NOI18N
                   replaceFirst("^org\\.netbeans\\.", "o.n."). // NOI18N
                   replaceFirst("^org\\.openide\\.", "openide."). // NOI18N
                   replaceFirst("^org\\.", "o."). // NOI18N
                   replaceFirst("^com\\.sun\\.", "c.s."). // NOI18N
                   replaceFirst("^com\\.", "c."); // NOI18N
    }
    
    /**
     * Find all modules in a binary build, possibly from cache.
     */
    private static Map<String,Entry> scanBinaries(Project project, File[] clusters) throws IOException {
        Map<String,Entry> allEntries = new HashMap<>();

        for (File cluster : clusters) {
            Map<String, Entry> entries = BINARY_SCAN_CACHE.get(cluster);
            if (entries == null) {
                if (project != null) {
                    project.log("Scanning for modules in " + cluster);
                }
                entries = new HashMap<>();
                doScanBinaries(cluster, entries);
                if (project != null) {
                    project.log("Found modules: " + entries.keySet(), Project.MSG_VERBOSE);
                }
                BINARY_SCAN_CACHE.put(cluster, entries);
            }
            allEntries.putAll(entries);
        }
        return allEntries;
    }
    
    private static final String[] MODULE_DIRS = {
        "modules",
        "modules/eager",
        "modules/autoload",
        "lib",
        "core",
    };
    /**
     * Look for all possible modules in a NB build.
     * Checks modules/{,autoload/,eager/}*.jar as well as well-known core/*.jar and lib/boot.jar in each cluster.
     * XXX would be slightly more precise to check config/Modules/*.xml rather than scan for module JARs.
     */
    private static void doScanBinaries(File cluster, Map<String,Entry> entries) throws IOException {
        File moduleAutoDepsDir = new File(new File(cluster, "config"), "ModuleAutoDeps");
            for (String moduleDir : MODULE_DIRS) {
                if (moduleDir.endsWith("lib") && !cluster.getName().contains("platform")) {
                    continue;
                }
                File dir = new File(cluster, moduleDir.replace('/', File.separatorChar));
                if (!dir.isDirectory()) {
                    continue;
                }
                File[] jars = dir.listFiles();
                if (jars == null) {
                    throw new IOException("Cannot examine dir " + dir);
                }
                for (File m : jars) {
                    scanOneBinary(m, cluster, entries, moduleAutoDepsDir);
                }
            }

            final File configDir = new File(new File(cluster, "config"), "Modules");
            File[] configs = configDir.listFiles();
            XPathExpression expr = null;
            DocumentBuilder b = null;
            if (configs != null) {
                for (File xml : configs) {
                    // TODO, read location, scan
                    final String fileName = xml.getName();
                    if (!fileName.endsWith(".xml")) {
                        continue;
                    }
                    if (entries.containsKey(fileName.substring(0, fileName.length() - 4).replace('-', '.'))) {
                        continue;
                    }
                    try {
                        if (expr == null) {
                            expr = XPathFactory.newInstance().newXPath().compile("/module/param[@name='jar']");
                            b = DocumentBuilderFactory.newInstance().newDocumentBuilder();
                            b.setEntityResolver(new EntityResolver() {
                                public InputSource resolveEntity(String publicId, String systemId)
                                throws SAXException, IOException {
                                    return new InputSource(new ByteArrayInputStream(new byte[0]));
                                }
                            });
                        }
                        Document doc = b.parse(xml);
                        String res = expr.evaluate(doc);
                        File jar = new File(cluster, res.replace('/', File.separatorChar));
                        if (!jar.isFile()) {
                            if (cluster.getName().equals("ergonomics")) {
                                // this is normal
                                continue;
                            }
                            throw new BuildException("Cannot find module " + jar + " from " + xml);
                        }
                        scanOneBinary(jar, cluster, entries, moduleAutoDepsDir);
                    } catch (Exception ex) {
                        throw new BuildException(ex);
                    }
                }
            }
    }
    
    private static Map<String,Entry> scanSuiteSources(Map<String,Object> properties, Project project) throws IOException {
        File basedir = new File((String) properties.get("basedir"));
        String suiteDir = (String) properties.get("suite.dir");
        if (suiteDir == null) {
            throw new IOException("No definition of suite.dir in " + basedir);
        }
        File suite = FileUtils.getFileUtils().resolveFile(basedir, suiteDir);
        if (!suite.isDirectory()) {
            throw new IOException("No such suite " + suite);
        }
        Map<String,Entry> entries = SUITE_SCAN_CACHE.get(suite);
        if (entries == null) {
            if (project != null) {
                project.log("Scanning for modules in suite " + suite);
            }
            entries = new HashMap<>();
            doScanSuite(entries, suite, properties, project);
            if (project != null) {
                project.log("Found modules: " + entries.keySet(), Project.MSG_VERBOSE);
            }
            SUITE_SCAN_CACHE.put(suite, entries);
        }
        return entries;
    }
    
    private static void doScanSuite(Map<String,Entry> entries, File suite, Map<String,Object> properties, Project project) throws IOException {
        Project fakeproj = new Project();
        fakeproj.setBaseDir(suite); // in case ${basedir} is used somewhere
        Property faketask = new Property();
        faketask.setProject(fakeproj);
        faketask.setFile(new File(suite, "nbproject/private/private.properties".replace('/', File.separatorChar)));
        faketask.execute();
        faketask.setFile(new File(suite, "nbproject/project.properties".replace('/', File.separatorChar)));
        faketask.execute();
        String modulesS = fakeproj.getProperty("modules");
        if (modulesS == null) {
            throw new IOException("No definition of modules in " + suite);
        }
        String[] modules = Path.translatePath(fakeproj, modulesS);
        for (int i = 0; i < modules.length; i++) {
            File module = new File(modules[i]);
            if (!module.isDirectory()) {
                throw new IOException("No such module " + module + " referred to from " + suite);
            }
            if (!scanPossibleProject(module, entries, properties, null, ModuleType.SUITE, project, null)) {
                throw new IOException("No valid module found in " + module + " referred to from " + suite);
            }
        }
    }
    
    private static Entry scanStandaloneSource(Map<String,Object> properties, Project project) throws IOException {
        if (properties.get("project") == null) return null; //Not a standalone module
        File basedir = new File((String) properties.get("project"));
        Entry entry = STANDALONE_SCAN_CACHE.get(basedir);
        if (entry == null) {
            Map<String,Entry> entries = new HashMap<>();
            if (!scanPossibleProject(basedir, entries, properties, null, ModuleType.STANDALONE, project, null)) {
                throw new IOException("No valid module found in " + basedir);
            }
            assert entries.size() == 1;
            entry = entries.values().iterator().next();
            STANDALONE_SCAN_CACHE.put(basedir, entry);
        }
        return entry;
    }
    
    /** all module entries, indexed by cnb */
    private final Map<String,Entry> entries;
    
    /**
     * Initiates scan if not already parsed.
     * Properties interpreted:
     * <ol>
     * <li> ${nb_all} - location of NB sources (used only for netbeans.org modules)
     * <li> ${netbeans.dest.dir} - location of NB build (used only for NB.org modules)
     * <li> ${cluster.path.final} - location of clusters to build against,
     * created from ${cluster.path} specified in platform.properties file (used only for suite and standalone modules)
     * <li> ${basedir} - directory of the project initiating the scan (most significant for standalone modules)
     * <li> ${suite.dir} - directory of the suite (used only for suite modules)
     * <li> ${nb.cluster.TOKEN} - list of module paths included in cluster TOKEN (comma-separated) (used only for netbeans.org modules)
     * <li> ${nb.cluster.TOKEN.dir} - directory in ${netbeans.dest.dir} where cluster TOKEN is built (used only for netbeans.org modules)
     * <li> ${project} - basedir for standalone modules
     * </ol>
     * @param properties some properties to be used (see above)
     * @param type the type of project
     * @param project a project ref, only for logging (may be null with no loss of semantics)
     */
    public ModuleListParser(Map<String,Object> properties, ModuleType type, Project project) throws IOException {
        String nball = (String) properties.get("nb_all");
        File basedir = new File((String) properties.get("basedir"));
        final FileUtils fu = FileUtils.getFileUtils();

        if (type != ModuleType.NB_ORG) {
            // add extra clusters
            String suiteDirS = (String) properties.get("suite.dir");
            boolean hasSuiteDir = suiteDirS != null && suiteDirS.length() > 0;
            String clusterPath = (String) properties.get("cluster.path.final");
            File[] clusters = null;

            if (clusterPath != null) {
                String[] clustersS;
                if (hasSuiteDir) {
                    // resolve suite modules against fake suite project
                    Project fakeproj = new Project();
                    fakeproj.setBaseDir(new File(suiteDirS));
                    clustersS = Path.translatePath(fakeproj, clusterPath);
                } else {
                    clustersS = Path.translatePath(project, clusterPath);
                }
                clusters = new File[clustersS.length];
                if (clustersS != null && clustersS.length > 0) {
                    for (int j = 0; j < clustersS.length; j++) {
                        File cluster = new File(clustersS[j]);
                        if (! cluster.isDirectory()) {
                            throw new IOException("No such cluster " + cluster + " referred to from ${cluster.path.final}: " + clusterPath);
                        }
                        clusters[j] = cluster;
                    }
                }
            }

            if (clusters == null || clusters.length == 0)
                throw new IOException("Invalid ${cluster.path.final}: " + clusterPath);

            // External module.
            if (nball != null && project != null) {
                project.log("You must *not* declare <suite-component/> or <standalone/> for a netbeans.org module in " + basedir + "; fix project.xml to use the /2 schema", Project.MSG_WARN);
            }
            entries = scanBinaries(project, clusters);
            if (type == ModuleType.SUITE) {
                entries.putAll(scanSuiteSources(properties, project));
            } else {
                assert type == ModuleType.STANDALONE;
                Entry e = scanStandaloneSource(properties, project);
                entries.put(e.getCnb(), e);
            }
        } else {
            // netbeans.org module.
            String buildS = (String) properties.get("netbeans.dest.dir");
            if (buildS == null) {
                throw new IOException("No definition of netbeans.dest.dir in " + basedir);
            }
            // Resolve against basedir, and normalize ../ sequences and so on in case they are used.
            // Neither operation is likely to be needed, but just in case.
            File build = fu.normalize(fu.resolveFile(basedir, buildS).getAbsolutePath());
            if (nball == null) {
                throw new IOException("You must declare either <suite-component/> or <standalone/> for an external module in " + new File((String) properties.get("basedir")));
            }
            String nbBuildDir = (String) properties.get("nb.build.dir");
            if (!build.equals(new File(new File(nball, "nbbuild"), "netbeans")) && !(nbBuildDir != null && build.equals(new File(nbBuildDir, "netbeans")))) {
                // Potentially orphaned module to be built against specific binaries, plus perhaps other source deps.
                if (!build.isDirectory()) {
                    throw new IOException("No such netbeans.dest.dir: " + build);
                }
                // expand clusters in build
                File[] clusters = build.listFiles();
                if (clusters == null) {
                    throw new IOException("Cannot examine dir " + build);
                }
                entries = scanBinaries(project, clusters);
                // Add referenced module in case it does not appear otherwise.
                Entry e = scanStandaloneSource(properties, project);
                if (e != null) {
                    entries.put(e.getCnb(), e);
                }
                entries.putAll(scanNetBeansOrgSources(new File(nball), properties, project));
            } else {
                entries = scanNetBeansOrgSources(new File(nball), properties, project);
            }
        }
    }
    /**
     * Find all entries in this list.
     * @return a set of all known entries
     */
    public Set<Entry> findAll() {
        return new HashSet<>(entries.values());
    }
    
    /**
     * Find one entry by code name base.
     * @param cnb the desired code name base
     * @return the matching entry or null
     */
    public Entry findByCodeNameBase(String cnb) {
        return entries.get(cnb);
    }

    
    /** parse Openide-Module-Module-Dependencies entry
     * @return array of code name bases
     */
    private static String[] parseRuntimeDependencies(String moduleDependencies) {
        if (moduleDependencies == null) {
            return new String[0];
        }
        List<String> cnds = new ArrayList<>();
        StringTokenizer toks = new StringTokenizer(moduleDependencies,",");
        while (toks.hasMoreTokens()) {
            String token = toks.nextToken().trim();
            // substring cnd/x
            int slIdx = token.indexOf('/');
            if (slIdx != -1) {
                token = token.substring(0,slIdx);
            }
            // substring cnd' 'xx
            slIdx = token.indexOf(' ');
            if (slIdx != -1) {
                token = token.substring(0,slIdx);
            }
            // substring cnd > 
            slIdx = token.indexOf('>');
            if (slIdx != -1) {
                token = token.substring(0,slIdx);
            }
            token = token.trim();
            if (token.length() > 0) {
               cnds.add(token);
            }
        }
        return cnds.toArray(new String[cnds.size()]);
    }

    static boolean scanOneBinary(File m, File cluster, Map<String, Entry> entries, File moduleAutoDepsDir) throws IOException {
        if (!m.getName().endsWith(".jar")) {
            return true;
        }
        JarFile jf;
        try {
            jf = new JarFile(m);
        } catch (IOException x) {
            throw new IOException("could not open " + m + ": " + x, x);
        }
        File dir = m.getParentFile();
        try {
            Attributes attr = jf.getManifest().getMainAttributes();
            String codename = JarWithModuleAttributes.extractCodeName(attr);
            if (codename == null) {
                return true;
            }
            String codenamebase;
            int slash = codename.lastIndexOf('/');
            if (slash == -1) {
                codenamebase = codename;
            } else {
                codenamebase = codename.substring(0, slash);
            }

            String cp = attr.getValue("Class-Path");
            File[] exts;
            if (cp == null) {
                exts = new File[0];
            } else {
                String[] pieces = cp.split(" +");
                exts = new File[pieces.length];
                for (int l = 0; l < pieces.length; l++) {
                    String append = URLDecoder.decode(pieces[l].replace('/', File.separatorChar), "UTF-8");
                    exts[l] = fixFxRtJar(new File(dir, append), null);
                }
            }
            String moduleDependencies = attr.getValue("OpenIDE-Module-Module-Dependencies");


            Entry entry = new Entry(codenamebase, m, exts, null, null, null, null, cluster.getName(),
                    parseRuntimeDependencies(moduleDependencies), Collections.<String,String[]>emptyMap(),
                    new File(moduleAutoDepsDir, codenamebase.replace('.', '-') + ".xml"));
            Entry prev = entries.put(codenamebase, entry);
            if (prev != null && !prev.equals(entry)) {
                throw new IOException("Duplicated module " + codenamebase + ": found in " + prev + " and " + entry);
            }
        } finally {
            jf.close();
        }
        return false;
    }
    
    /**
     * One entry in the file.
     */
    @SuppressWarnings("serial") // really want it to be incompatible if format changes
    public static final class Entry implements Serializable {

        private final String cnb;
        private final File jar;
        private final File[] classPathExtensions;
        private final File sourceLocation;
        private final String netbeansOrgPath;
        private final String netbeansOrgId;
        private final String[] buildPrerequisites;
        private final String clusterName;
        private final String[] runtimeDependencies; 
        // dependencies on other tests
        private final Map<String,String[]> testDependencies;
        private final File moduleAutoDeps;
        
        Entry(String cnb, File jar, File[] classPathExtensions, File sourceLocation, String netbeansOrgPath, String netbeansOrgId,
                String[] buildPrerequisites, String clusterName,String[] runtimeDependencies, Map<String,String[]> testDependencies, File moduleAutoDeps) {
            this.cnb = cnb;
            this.jar = jar;
            this.classPathExtensions = classPathExtensions;
            this.sourceLocation = sourceLocation;
            this.netbeansOrgPath = netbeansOrgPath;
            this.netbeansOrgId = netbeansOrgId;
            this.buildPrerequisites = buildPrerequisites;
            this.clusterName = clusterName;
            this.runtimeDependencies = runtimeDependencies;
            assert testDependencies != null;
            this.testDependencies = testDependencies;
            this.moduleAutoDeps = moduleAutoDeps;
        }
        
        /**
         * Get the code name base, e.g. org.netbeans.modules.ant.grammar.
         */
        public String getCnb() {
            return cnb;
        }
        
        /**
         * Get the absolute JAR location, e.g. .../ide/modules/org-netbeans-modules-ant-grammar.jar.
         */
        public File getJar() {
            return jar;
        }
        
        /**
         * Get a list of extensions to the class path of this module (may be empty).
         */
        public File[] getClassPathExtensions() {
            return classPathExtensions;
        }

        /**
         * @return the sourceLocation, may be null.
         */
        public File getSourceLocation() {
            return sourceLocation;
        }
        
        /**
         * Get the path within netbeans.org, if this is a netbeans.org module (else null).
         */
        public String getNetbeansOrgPath() {
            return netbeansOrgPath;
        }
        
        /**
         * Get a list of declared build prerequisites (or null for sourceless entries).
         * Each entry is a code name base.
         */
        public String[] getBuildPrerequisites() {
            return buildPrerequisites;
        }
        /** Get runtime dependencies, OpenIDE-Module-Dependencies entry. 
         * Each entry is a code name base.
         */
        public String[] getRuntimeDependencies() {
            return runtimeDependencies;
        }
        
        /**
         * Return the name of the cluster in which this module resides.
         * If this entry represents an external module in source form,
         * then the cluster will be null. If the module represents a netbeans.org
         * module or a binary module in a platform, then the cluster name will
         * be the (base) name of the directory containing the "modules" subdirectory
         * (sometimes "lib" or "core") where the JAR is.
         */
        public String getClusterName() {
            return clusterName;
        }
        
        public Map<String,String[]> getTestDependencies() {
            return testDependencies;
        }

        /**
         * Gets the module auto dependencies XML file.
         * @return a file location (may or may not exist)
         */
        public File getModuleAutoDeps() {
            return moduleAutoDeps;
        }

        public @Override String toString() {
            return (sourceLocation != null ? sourceLocation : jar).getAbsolutePath();
        }

        @Override
        public boolean equals(Object obj) {
            if (obj == null) {
                return false;
            }
            if (getClass() != obj.getClass()) {
                return false;
            }
            final Entry other = (Entry) obj;
            if ((this.cnb == null) ? (other.cnb != null) : !this.cnb.equals(other.cnb)) {
                return false;
            }
            if (this.jar != other.jar && (this.jar == null || !this.jar.equals(other.jar))) {
                return false;
            }
            if (!Arrays.deepEquals(this.classPathExtensions, other.classPathExtensions)) {
                return false;
            }
            if (this.sourceLocation != other.sourceLocation && (this.sourceLocation == null || !this.sourceLocation.equals(other.sourceLocation))) {
                return false;
            }
            if ((this.netbeansOrgPath == null) ? (other.netbeansOrgPath != null) : !this.netbeansOrgPath.equals(other.netbeansOrgPath)) {
                return false;
            }
            if ((this.netbeansOrgId == null) ? (other.netbeansOrgId != null) : !this.netbeansOrgId.equals(other.netbeansOrgId)) {
                return false;
            }
            if (!Arrays.deepEquals(this.buildPrerequisites, other.buildPrerequisites)) {
                return false;
            }
            if ((this.clusterName == null) ? (other.clusterName != null) : !this.clusterName.equals(other.clusterName)) {
                return false;
            }
            if (!Arrays.deepEquals(this.runtimeDependencies, other.runtimeDependencies)) {
                return false;
            }
            if (this.testDependencies != other.testDependencies && (this.testDependencies == null || !this.testDependencies.equals(other.testDependencies))) {
                return false;
            }
            return true;
        }

        @Override
        public int hashCode() {
            int hash = 5;
            hash = 83 * hash + (this.cnb != null ? this.cnb.hashCode() : 0);
            hash = 83 * hash + (this.jar != null ? this.jar.hashCode() : 0);
            hash = 83 * hash + Arrays.deepHashCode(this.classPathExtensions);
            hash = 83 * hash + (this.sourceLocation != null ? this.sourceLocation.hashCode() : 0);
            hash = 83 * hash + (this.netbeansOrgPath != null ? this.netbeansOrgPath.hashCode() : 0);
            hash = 83 * hash + (this.netbeansOrgId != null ? this.netbeansOrgId.hashCode() : 0);
            hash = 83 * hash + Arrays.deepHashCode(this.buildPrerequisites);
            hash = 83 * hash + (this.clusterName != null ? this.clusterName.hashCode() : 0);
            hash = 83 * hash + Arrays.deepHashCode(this.runtimeDependencies);
            hash = 83 * hash + (this.testDependencies != null ? this.testDependencies.hashCode() : 0);
            return hash;
        }

        String getNetbeansOrgId() {
            return netbeansOrgId;
        }


    }

}
