// mixin implementing the buildIdealTree method
const localeCompare = require('@isaacs/string-locale-compare')('en')
const PackageJson = require('@npmcli/package-json')
const npa = require('npm-package-arg')
const pacote = require('pacote')
const cacache = require('cacache')
const { callLimit: promiseCallLimit } = require('promise-call-limit')
const realpath = require('../../lib/realpath.js')
const { resolve, dirname, sep } = require('node:path')
const treeCheck = require('../tree-check.js')
const { readdirScoped } = require('@npmcli/fs')
const { lstat, readlink } = require('node:fs/promises')
const { depth } = require('treeverse')
const { log, time } = require('proc-log')
const { redact } = require('@npmcli/redact')

const {
  OK,
  REPLACE,
  CONFLICT,
} = require('../can-place-dep.js')
const PlaceDep = require('../place-dep.js')

const debug = require('../debug.js')
const fromPath = require('../from-path.js')
const calcDepFlags = require('../calc-dep-flags.js')
const { isReleaseAgeExcluded, trustedSpecName } = require('../release-age-exclude.js')
const { resolvePatchedDependencies } = require('../patched-dependencies.js')
const PackageExtensions = require('../package-extensions.js')
const NpmExtension = require('../npm-extension.js')
const { hasExtensionFile } = require('../npm-extension.js')
const Shrinkwrap = require('../shrinkwrap.js')
const { defaultLockfileVersion } = Shrinkwrap
const Node = require('../node.js')
const Link = require('../link.js')

// Maps a parsed spec.type to the corresponding allow-* arborist option name.
// Hoisted to module scope so #checkAllow doesn't re-allocate it per call.
const ALLOW_OPTION_FOR_TYPE = {
  git: 'allowGit',
  remote: 'allowRemote',
  file: 'allowFile',
  directory: 'allowDirectory',
}
const addRmPkgDeps = require('../add-rm-pkg-deps.js')
const optionalSet = require('../optional-set.js')
const { checkEngine, checkPlatform } = require('npm-install-checks')
const relpath = require('../relpath.js')
const resetDepFlags = require('../reset-dep-flags.js')

// note: some of these symbols are shared so we can hit
// them with unit tests and reuse them across mixins
const _updateAll = Symbol.for('updateAll')
const _flagsSuspect = Symbol.for('flagsSuspect')
const _setWorkspaces = Symbol.for('setWorkspaces')
const _updateNames = Symbol.for('updateNames')
const _resolvedAdd = Symbol.for('resolvedAdd')
const _rpcache = Symbol.for('realpathCache')
const _stcache = Symbol.for('statCache')

// used by Reify mixin
const _addNodeToTrashList = Symbol.for('addNodeToTrashList')

// Push items in, pop them sorted by depth and then path
// Sorts physically shallower deps up to the front of the queue, because
// they'll affect things deeper in, then alphabetical for consistency between
// installs
class DepsQueue {
  #deps = []
  #sorted = true

  get length () {
    return this.#deps.length
  }

  push (item) {
    if (!this.#deps.includes(item)) {
      this.#sorted = false
      this.#deps.push(item)
    }
  }

  pop () {
    if (!this.#sorted) {
      this.#deps.sort((a, b) => (a.depth - b.depth) || localeCompare(a.path, b.path))
      this.#sorted = true
    }
    return this.#deps.shift()
  }
}

module.exports = cls => class IdealTreeBuilder extends cls {
  #complete
  #currentDep = null
  #depsQueue = new DepsQueue()
  #depsSeen = new Set()
  #explicitRequests = new Set()
  #follow
  #installStrategy
  #linkNodes = new Set()
  #loadFailures = new Set()
  #manifests = new Map()
  #mutateTree = false
  #packageExtensions = null
  #npmExtension = null
  #requestedTreeMutation = false
  // a map of each module in a peer set to the thing that depended on
  // that set of peers in the first place.  Use a WeakMap so that we
  // don't hold onto references for nodes that are garbage collected.
  #peerSetSource = new WeakMap()
  #preferDedupe = false
  #prune
  #strictPeerDeps
  #virtualRoots = new Map()

  constructor (options) {
    super(options)

    const {
      follow = false,
      installStrategy = 'hoisted',
      strictPeerDeps = false,
      global,
    } = options

    this.#strictPeerDeps = !!strictPeerDeps

    this.#installStrategy = global ? 'shallow' : installStrategy
    this.#follow = !!follow

    this[_updateAll] = false
    this[_updateNames] = []
    this[_resolvedAdd] = []

    // caches for cached realpath calls
    const cwd = process.cwd()
    // assume that the cwd is real enough for our purposes
    this[_rpcache] = new Map([[cwd, cwd]])
    this[_stcache] = new Map()
    this[_flagsSuspect] = false
  }

  get explicitRequests () {
    return new Set(this.#explicitRequests)
  }

  // public method
  async buildIdealTree (options = {}) {
    if (this.idealTree) {
      return this.idealTree
    }

    // allow the user to set reify options on the ctor as well.
    // XXX: deprecate separate reify() options object.
    options = { ...this.options, ...options }

    // an empty array or any falsey value is the same as null
    if (!options.add || options.add.length === 0) {
      options.add = null
    }
    if (!options.rm || options.rm.length === 0) {
      options.rm = null
    }

    const timeEnd = time.start('idealTree')

    if (!options.add && !options.rm && !options.update && this.options.global) {
      throw new Error('global requires add, rm, or update option')
    }

    // first get the virtual tree, if possible.  If there's a lockfile, then
    // that defines the ideal tree, unless the root package.json is not
    // satisfied by what the ideal tree provides.
    // from there, we start adding nodes to it to satisfy the deps requested
    // by the package.json in the root.

    this.#parseSettings(options)

    // start tracker block
    this.addTracker('idealTree')

    try {
      await this.#initTree()
      await this.#loadNpmExtension()
      this.#loadPackageExtensions()
      await this.#inflateAncientLockfile()
      await this.#applyUserRequests(options)
      await this.#buildDeps()
      await this.#fixDepFlags()
      await this.#pruneFailedOptional()
      await this.#checkEngineAndPlatform()
      await resolvePatchedDependencies(this.idealTree, {
        path: this.path,
        allowUnusedPatches: this.options.allowUnusedPatches,
        rm: options.rm || [],
      })
      this.#warnWorkspacePackageExtensions()
      this.#warnWorkspaceNpmExtension()
    } finally {
      timeEnd()
      this.finishTracker('idealTree')
    }

    return treeCheck(this.idealTree)
  }

  async #checkEngineAndPlatform () {
    const { engineStrict, npmVersion, nodeVersion, omit = [], cpu, os, libc } = this.options
    const omitSet = new Set(omit)

    for (const node of this.idealTree.inventory.values()) {
      if (!node.optional && !node.shouldOmit(omitSet)) {
        try {
          // if devEngines is present in the root node we ignore the engines check
          if (!(node.isRoot && node.package.devEngines)) {
            checkEngine(node.package, npmVersion, nodeVersion, this.options.force)
          }
        } catch (err) {
          if (engineStrict) {
            throw err
          }
          log.warn(err.code, err.message, {
            package: err.pkgid,
            required: err.required,
            current: err.current,
          })
        }
        checkPlatform(node.package, this.options.force)
      }
      if (node.optional && !node.inert) {
        // Mark any optional packages we can't install as inert.
        // We ignore the --force and --engine-strict flags.
        try {
          checkEngine(node.package, npmVersion, nodeVersion, false)
          checkPlatform(node.package, false, { cpu, os, libc })
        } catch (error) {
          const set = optionalSet(node)
          for (const node of set) {
            node.inert = true
          }
        }
      }
    }
  }

  // Load the root project's packageExtensions rule set.
  // Only the workspace root is authoritative, matching the root-only model of overrides.
  // The canonical hash is stashed on the lockfile meta so commit() can persist it.
  #loadPackageExtensions () {
    const rootPkg = this.idealTree.target.package
    const lockedHash = this.idealTree.meta.packageExtensionsHash
    this.#packageExtensions = new PackageExtensions(rootPkg.packageExtensions)
    this.idealTree.meta.packageExtensionsHash = this.#packageExtensions.hash

    // When the rule set has changed since the lockfile was written, the locked manifests for affected packages are stale.
    // The locked manifest is the effective, already-extended manifest, so detach those nodes and rebuild them from fresh manifests under the current rules.
    if (this.idealTree.meta.loadedFromDisk && lockedHash !== this.#packageExtensions.hash) {
      for (const node of [...this.idealTree.inventory.values()]) {
        if (node.isProjectRoot || node.isWorkspace || node.isTop) {
          continue
        }
        // a node is affected if it carries provenance from the old rules or matches a current selector
        const affected = node.packageExtensionsApplied ||
          this.#packageExtensions.wouldMatch(node.packageName, node.version)
        if (affected) {
          for (const edge of node.edgesIn) {
            this.#depsQueue.push(edge.from)
          }
          node.parent = null
        }
      }
    }
  }

  // Apply a matching root packageExtension to a copy of a candidate manifest.
  // Returns the possibly-extended manifest and the provenance to attach to the node.
  // Workspace candidates are never extended; that warning is emitted separately.
  #applyPackageExtension (pkg) {
    if (!this.#packageExtensions?.present) {
      return { pkg, applied: null }
    }
    const res = this.#packageExtensions.apply(pkg)
    return res ? { pkg: res.pkg, applied: res.applied } : { pkg, applied: null }
  }

  // Load the root project's .npm-extension file and its transformManifest export.
  // ignore-extension (and, via flatten, ignore-scripts) disables discovery and execution; the lockfile then carries no extension state.
  // The selected file's hash is stashed on the lockfile meta so commit() can persist it and npm ci can detect stale extension state.
  async #loadNpmExtension () {
    const lockedHash = this.idealTree.meta.npmExtensionHash
    // ignore-extension (and, via flatten, ignore-scripts) disables discovery and execution.
    // Leave any locked extension state untouched so npm ci reifies the locked graph as-is and the lockfile stays internally consistent.
    if (this.options.ignoreExtension) {
      return
    }
    const ext = new NpmExtension({
      root: this.idealTree.realpath,
      extensionFile: this.options.extensionFile,
    })
    this.#npmExtension = ext
    this.idealTree.meta.npmExtensionHash = ext.hash
    if (ext.present) {
      await ext.load()
    }

    // When the extension file changed since the lockfile was written, locked manifests may no longer reflect its output.
    // This also covers removal: a now-absent file hashes to null, which differs from the locked hash and reverts the previously transformed nodes.
    // Arbitrary code has no selector to predict which packages it affects, so re-resolve any node that carried old provenance or that the current file now transforms.
    if (this.idealTree.meta.loadedFromDisk && lockedHash !== ext.hash) {
      for (const node of [...this.idealTree.inventory.values()]) {
        if (node.isProjectRoot || node.isWorkspace || node.isTop || node.isLink) {
          continue
        }
        // A node with old provenance carries an already-transformed manifest, so always refetch it.
        // Otherwise probe the locked manifest (without caching, so the authoritative full-manifest fetch is not pre-seeded).
        // The locked manifest carries every resolution-relevant field (name, version, dependencies, peer metadata); a transform that newly targets a node by a field not persisted in the lockfile is the documented edge that may need a manual re-lock.
        const affected = node.npmExtensionApplied || ext.apply(node.package, { memoize: false })
        if (affected) {
          for (const edge of node.edgesIn) {
            this.#depsQueue.push(edge.from)
          }
          node.parent = null
        }
      }
    }
  }

  // Apply the root transformManifest to a copy of a candidate manifest, before any packageExtensions rule.
  // Returns the possibly-transformed manifest and the provenance to attach to the node.
  #applyNpmExtension (pkg) {
    const res = this.#npmExtension?.apply(pkg)
    return res ? { pkg: res.pkg, applied: res.applied } : { pkg, applied: null }
  }

  // Warn when packageExtensions appears in a non-root workspace, or when a root selector matches a workspace member.
  // Workspace package manifests are edited directly and are never extension targets.
  #warnWorkspacePackageExtensions () {
    if (!this.#packageExtensions?.present) {
      return
    }
    for (const node of this.idealTree.inventory.values()) {
      // a workspace is in the inventory as both a Link and its target node; warn once by skipping the link
      if (!node.isWorkspace || node.isLink) {
        continue
      }
      if (node.package.packageExtensions !== undefined) {
        log.warn('packageExtensions',
          `"packageExtensions" in workspace ${node.name} is ignored; it is only honored at the workspace root`)
      }
      if (this.#packageExtensions.wouldMatch(node.name, node.version)) {
        log.warn('packageExtensions',
          `selector matches workspace package ${node.name}@${node.version}; edit its package.json directly instead of using packageExtensions`)
      }
    }
  }

  // Warn when a non-root workspace package contains a .npm-extension file; only the workspace root's file is honored.
  #warnWorkspaceNpmExtension () {
    for (const node of this.idealTree.inventory.values()) {
      // a workspace is in the inventory as both a Link and its target node; warn once by skipping the link
      if (!node.isWorkspace || node.isLink || node.isProjectRoot) {
        continue
      }
      if (hasExtensionFile(node.realpath)) {
        log.warn('npm-extension',
          `".npm-extension" in workspace ${node.name} is ignored; it is only honored at the workspace root`)
      }
    }
  }

  #parseSettings (options) {
    const update = options.update === true ? { all: true }
      : Array.isArray(options.update) ? { names: options.update }
      : options.update || {}

    if (update.all || !Array.isArray(update.names)) {
      update.names = []
    }

    this.#complete = !!options.complete
    this.#preferDedupe = !!options.preferDedupe

    // validates list of update names, they must
    // be dep names only, no semver ranges are supported
    for (const name of update.names) {
      const spec = npa(name)
      const validationError =
        new TypeError(`Update arguments must only contain package names, eg:
    npm update ${spec.name}`)
      validationError.code = 'EUPDATEARGS'

      // If they gave us anything other than a bare package name
      if (spec.raw !== spec.name) {
        throw validationError
      }
    }
    this[_updateNames] = update.names

    this[_updateAll] = update.all
    // we prune by default unless explicitly set to boolean false
    this.#prune = options.prune !== false

    // set if we add anything, but also set here if we know we'll make
    // changes and thus have to maybe prune later.
    this.#requestedTreeMutation = !!(
      options.add ||
      options.rm ||
      update.all ||
      update.names.length
    )
    this.#mutateTree = this.#requestedTreeMutation
  }

  // load the initial tree, either the virtualTree from a shrinkwrap,
  // or just the root node from a package.json
  async #initTree () {
    const timeEnd = time.start('idealTree:init')
    let root
    if (this.options.global) {
      root = await this.#globalRootNode()
    } else {
      try {
        const { content: pkg } = await PackageJson.normalize(this.path)
        root = await this.#rootNodeFromPackage(pkg)
      } catch (err) {
        if (err.code === 'EJSONPARSE') {
          throw err
        }
        root = await this.#rootNodeFromPackage({})
      }
    }
    return this[_setWorkspaces](root)
      // ok to not have a virtual tree.  probably initial install.
      // When updating all, we load the shrinkwrap, but don't bother
      // to build out the full virtual tree from it, since we'll be
      // reconstructing it anyway.
      .then(root => {
        if (this.options.global) {
          return root
        } else if (!this.options.usePackageLock || this[_updateAll]) {
          return Shrinkwrap.reset({
            path: this.path,
            lockfileVersion: this.options.lockfileVersion,
            resolveOptions: this.options,
          }).then(meta => Object.assign(root, { meta }))
        } else {
          return this.loadVirtual({ root })
        }
      })

      // if we don't have a lockfile to go from, then start with the
      // actual tree, so we only make the minimum required changes.
      // don't do this for global installs or updates, because in those
      // cases we don't use a lockfile anyway.
      // Load on a new Arborist object, so the Nodes aren't the same,
      // or else it'll get super confusing when we change them!
      .then(async root => {
        if ((!this[_updateAll] && !this.options.global && !root.meta.loadedFromDisk) || (this.options.global && this[_updateNames].length)) {
          await new this.constructor(this.options).loadActual({ root })
          const tree = root.target
          // even though we didn't load it from a package-lock.json FILE,
          // we still loaded it "from disk", meaning we have to reset
          // dep flags before assuming that any mutations were reflected.
          if (tree.children.size) {
            root.meta.loadedFromDisk = true
            // set these so that we don't try to ancient lockfile reload it
            root.meta.originalLockfileVersion = root.meta.lockfileVersion = this.options.lockfileVersion || defaultLockfileVersion
          }
        }
        root.meta.inferFormattingOptions(root.package)
        return root
      })

      .then(tree => {
        // search the virtual tree for missing/invalid edges, if any are found add their source to
        // the depsQueue so that we'll fix it later
        depth({
          tree,
          getChildren: (node) => {
            const children = []
            for (const edge of node.edgesOut.values()) {
              children.push(edge.to)
            }
            return children
          },
          filter: node => node,
          visit: node => {
            for (const edge of node.edgesOut.values()) {
              const skipPeerOptional = edge.type === 'peerOptional' &&
                this.options.save === false &&
                !this.#requestedTreeMutation
              if (!skipPeerOptional && (!edge.to || !edge.valid)) {
                this.#depsQueue.push(node)
                break // no need to continue the loop after the first hit
              }
            }
          },
        })
        // null the virtual tree, because we're about to hack away at it
        // if you want another one, load another copy.
        this.idealTree = tree
        this.virtualTree = null
        timeEnd()
        return tree
      })
  }

  async #globalRootNode () {
    const root = await this.#rootNodeFromPackage({ dependencies: {} })
    // this is a gross kludge to handle the fact that we don't save
    // metadata on the root node in global installs, because the "root"
    // node is something like /usr/local/lib.
    const meta = new Shrinkwrap({
      path: this.path,
      lockfileVersion: this.options.lockfileVersion,
      resolveOptions: this.options,
    })
    meta.reset()
    root.meta = meta
    return root
  }

  async #rootNodeFromPackage (pkg) {
    // if the path doesn't exist, then we explode at this point. Note that
    // this is not a problem for reify(), since it creates the root path
    // before ever loading trees.
    // TODO: make buildIdealTree() and loadActual handle a missing root path,
    // or a symlink to a missing target, and let reify() create it as needed.
    const real = await realpath(this.path, this[_rpcache], this[_stcache])
    const Cls = real === this.path ? Node : Link
    const root = new Cls({
      path: this.path,
      realpath: real,
      pkg,
      extraneous: false,
      dev: false,
      devOptional: false,
      peer: false,
      optional: false,
      global: this.options.global,
      installLinks: this.installLinks,
      legacyPeerDeps: this.legacyPeerDeps,
      loadOverrides: true,
    })
    if (root.isLink) {
      root.target = new Node({
        path: real,
        realpath: real,
        pkg,
        extraneous: false,
        dev: false,
        devOptional: false,
        peer: false,
        optional: false,
        global: this.options.global,
        installLinks: this.installLinks,
        legacyPeerDeps: this.legacyPeerDeps,
        loadOverrides: true,
        root,
      })
    }
    return root
  }

  // process the add/rm requests by modifying the root node, and the
  // update.names request by queueing nodes dependent on those named.
  async #applyUserRequests (options) {
    const timeEnd = time.start('idealTree:userRequests')
    const tree = this.idealTree.target

    if (!this.options.workspaces.length) {
      await this.#applyUserRequestsToNode(tree, options)
    } else {
      const nodes = this.workspaceNodes(tree, this.options.workspaces)
      if (this.options.includeWorkspaceRoot) {
        nodes.push(tree)
      }
      const appliedRequests = nodes.map(
        node => this.#applyUserRequestsToNode(node, options)
      )
      await Promise.all(appliedRequests)
    }

    timeEnd()
  }

  async #applyUserRequestsToNode (tree, options) {
    // If we have a list of package names to update, and we know it's
    // going to update them wherever they are, add any paths into those
    // named nodes to the buildIdealTree queue.
    if (!this.options.global && this[_updateNames].length) {
      this.#queueNamedUpdates()
    }

    // global updates only update the globalTop nodes, but we need to know
    // that they're there, and not reinstall the world unnecessarily.
    const globalExplicitUpdateNames = []
    if (this.options.global && (this[_updateAll] || this[_updateNames].length)) {
      const nm = resolve(this.path, 'node_modules')
      const paths = await readdirScoped(nm).catch(() => [])
      for (const p of paths) {
        const name = p.replace(/\\/g, '/')
        // Match loadActual behavior: hidden entries and retired scoped package
        // folders are not installed global packages.
        if (/^(@[^/]+\/)?\./.test(name)) {
          continue
        }
        const updateName = this[_updateNames].includes(name)
        if (this[_updateAll] || updateName) {
          if (updateName) {
            globalExplicitUpdateNames.push(name)
          }
          const dir = resolve(nm, name)
          const st = await lstat(dir)
            .catch(/* istanbul ignore next */ () => null)
          if (st && st.isSymbolicLink()) {
            const target = await readlink(dir)
            const real = resolve(dirname(dir), target)
            tree.package.dependencies[name] = `file:${real}`
          } else {
            tree.package.dependencies[name] = '*'
          }
        }
      }
    }

    if (this.auditReport && this.auditReport.size > 0) {
      await this.#queueVulnDependents(options)
    }

    const { add, rm } = options

    if (rm && rm.length) {
      addRmPkgDeps.rm(tree.package, rm)
      for (const name of rm) {
        this.#explicitRequests.add({ from: tree, name, action: 'DELETE' })
      }
    }

    if (add && add.length) {
      await this.#add(tree, options)
    }

    // triggers a refresh of all edgesOut.  this has to be done BEFORE
    // adding the edges to explicitRequests, because the package setter
    // resets all edgesOut.
    if (add && add.length || rm && rm.length || this.options.global) {
      tree.package = tree.package
    }

    for (const spec of this[_resolvedAdd]) {
      if (spec.tree === tree) {
        this.#explicitRequests.add(tree.edgesOut.get(spec.name))
      }
    }
    for (const name of globalExplicitUpdateNames) {
      this.#explicitRequests.add(tree.edgesOut.get(name))
    }

    this.#depsQueue.push(tree)
  }

  // This returns a promise because we might not have the name yet, and need to call pacote.manifest to find the name.
  async #add (tree, { add, saveType = null, saveBundle = false }) {
    // If we have a link it will need to be added relative to the target's path
    const path = tree.target.path

    // Get the name for each of the specs in the list.
    // e.g. doing `foo@bar` we just return foo but if it's a url or git, we don't know the name until we fetch it and look in its manifest.
    await Promise.all(add.map(async rawSpec => {
      // We do NOT provide the path to npa here, because user-additions need to be resolved relative to the tree being added to.
      let spec = npa(rawSpec)

      // if it's just @'' then we reload whatever's there, or get latest.
      // if it's an explicit tag, we need to install that specific tag version.
      const isTag = spec.rawSpec && spec.type === 'tag'

      // look up the names of file/directory/git specs
      if (!spec.name || isTag) {
        const _isRoot = tree.isProjectRoot || tree.isWorkspace
        const mani = await pacote.manifest(spec, {
          ...this.options,
          _isRoot,
          before: this.#releaseAgeBefore(spec),
        })
        if (isTag) {
          // translate tag to a version
          spec = npa(`${mani.name}@${mani.version}`)
        }
        spec.name = mani.name
      }

      const { name } = spec
      if (spec.type === 'file') {
        spec = npa(`file:${relpath(path, spec.fetchSpec)}`, path)
        spec.name = name
      } else if (spec.type === 'directory') {
        try {
          const real = await realpath(spec.fetchSpec, this[_rpcache], this[_stcache])
          spec = npa(`file:${relpath(path, real)}`, path)
          spec.name = name
        } catch {
          // TODO: create synthetic test case to simulate realpath failure
        }
      }
      spec.tree = tree
      this[_resolvedAdd].push(spec)
    }))

    // now this._resolvedAdd is a list of spec objects with names.
    // find a home for each of them!
    addRmPkgDeps.add({
      pkg: tree.package,
      add: this[_resolvedAdd],
      saveBundle,
      saveType,
    })
  }

  // TODO: provide a way to fix bundled deps by exposing metadata about
  // what's in the bundle at each published manifest.  Without that, we
  // can't possibly fix bundled deps without breaking a ton of other stuff,
  // and leaving the user subject to getting it overwritten later anyway.
  async #queueVulnDependents (options) {
    for (const vuln of this.auditReport.values()) {
      // A fix is available in-range but a release-age window blocks the patched
      // version, so audit fix leaves this package at a vulnerable version.
      if (vuln.fixBlockedByReleaseAge) {
        const { version, before } = vuln.fixBlockedByReleaseAge
        const cutoff = new Date(before).toISOString().slice(0, 10)
        log.warn('audit', `A fix for ${vuln.name} is available (${vuln.name}@${version}) ` +
          `but was published after the configured release-age cutoff (${cutoff}), so ` +
          `${vuln.name} was left at a vulnerable version.\n` +
          `To install it, add "${vuln.name}" to min-release-age-exclude, or relax ` +
          'min-release-age or before.')
      }
      for (const node of vuln.nodes) {
        const bundler = node.getBundler()

        // XXX this belongs in the audit report itself, not here.
        // We shouldn't even get these things here, and they shouldn't
        // be printed by npm-audit-report as if they can be fixed, because
        // they can't.
        if (bundler) {
          log.warn(`audit fix ${node.name}@${node.version}`,
            `${node.location}\nis a bundled dependency of\n${
            bundler.name}@${bundler.version} at ${bundler.location}\n` +
            'It cannot be fixed automatically.\n' +
            `Check for updates to the ${bundler.name} package.`)
          continue
        }

        for (const edge of node.edgesIn) {
          this.addTracker('idealTree', edge.from.name, edge.from.location)
          this.#depsQueue.push(edge.from)
        }
      }
    }

    // note any that can't be fixed at the root level without --force
    // if there's a fix, we use that.  otherwise, the user has to remove it,
    // find a different thing, fix the upstream, etc.
    //
    // XXX: how to handle top nodes that aren't the root?  Maybe the report
    // just tells the user to cd into that directory and fix it?
    if (this.options.force && this.auditReport && this.auditReport.topVulns.size) {
      options.add = options.add || []
      options.rm = options.rm || []
      const nodesTouched = new Set()
      for (const [name, topVuln] of this.auditReport.topVulns.entries()) {
        const {
          simpleRange,
          topNodes,
          fixAvailable,
        } = topVuln
        for (const node of topNodes) {
          if (!node.isProjectRoot && !node.isWorkspace) {
            // not something we're going to fix, sorry.  have to cd into
            // that directory and fix it yourself.
            log.warn('audit', 'Manual fix required in linked project ' +
              `at ./${node.location} for ${name}@${simpleRange}.\n` +
              `'cd ./${node.location}' and run 'npm audit' for details.`)
            continue
          }

          if (!fixAvailable) {
            log.warn('audit', `No fix available for ${name}@${simpleRange}`)
            continue
          }

          // name may be different if parent fixes the dep
          // see Vuln fixAvailable setter
          const { isSemVerMajor, version, name: fixName } = fixAvailable
          const breakingMessage = isSemVerMajor
            ? 'a SemVer major change'
            : 'outside your stated dependency range'
          log.warn('audit', `Updating ${fixName} to ${version}, ` +
            `which is ${breakingMessage}.`)

          await this.#add(node, { add: [`${fixName}@${version}`] })
          nodesTouched.add(node)
        }
      }
      for (const node of nodesTouched) {
        node.package = node.package
      }
    }
  }

  #avoidRange (name) {
    if (!this.auditReport) {
      return null
    }
    const vuln = this.auditReport.get(name)
    if (!vuln) {
      return null
    }
    return vuln.range
  }

  // Enforces the allow-git / allow-file / allow-directory / allow-remote configs at the arborist resolution layer, before any branching into the symlink (Link) path or the manifest-fetch path.
  // Pacote also enforces these inside FetcherBase.get() as defense-in-depth, but the symlink branch never reaches pacote, and the manifest cache here would bypass pacote on a cached hit.
  // Throws the same { code: EALLOW${TYPE} } shape pacote uses, so callers and downstream consumers stay consistent.
  #checkAllow (spec, edge) {
    const optName = ALLOW_OPTION_FOR_TYPE[spec.type]
    if (!optName) {
      return
    }
    const allow = this.options[optName] ?? 'all'
    if (allow === 'all') {
      return
    }
    const isRoot = !!(edge?.from?.isProjectRoot || edge?.from?.isWorkspace)
    if (allow !== 'none' && isRoot) {
      return
    }
    throw Object.assign(
      new Error(`Fetching${allow === 'root' ? ' non-root' : ''} packages of type "${spec.type}" have been disabled`),
      {
        code: `EALLOW${spec.type.toUpperCase()}`,
        package: spec.toString(),
      }
    )
  }

  // Builds a Node representing a spec we failed to load (allow-* gate, network failure, ENOTARGET, etc.) and records it in #loadFailures so #pruneFailedOptional can later decide whether the failure is fatal or silently dropped for optional deps.
  #failureNode (name, parent, error, edge) {
    error.requiredBy = edge?.from?.location || '.'
    const n = new Node({
      name,
      parent,
      error,
      installLinks: this.installLinks,
      legacyPeerDeps: this.legacyPeerDeps,
    })
    this.#loadFailures.add(n)
    return n
  }

  #queueNamedUpdates () {
    // ignore top nodes, since they are not loaded the same way, and
    // probably have their own project associated with them.

    // for every node with one of the names on the list, we add its
    // dependents to the queue to be evaluated.  in buildDepStep,
    // anything on the update names list will get refreshed, even if
    // it isn't a problem.

    // XXX this could be faster by doing a series of inventory.query('name')
    // calls rather than walking over everything in the tree.
    for (const node of this.idealTree.inventory.values()) {
      // XXX add any invalid edgesOut to the queue
      if (this[_updateNames].includes(node.name) &&
        !node.isTop && !node.inDepBundle) {
        for (const edge of node.edgesIn) {
          this.addTracker('idealTree', edge.from.name, edge.from.location)
          this.#depsQueue.push(edge.from)
        }
      }
    }
  }

  async #inflateAncientLockfile () {
    const { meta, inventory } = this.idealTree
    const ancient = meta.ancientLockfile
    const old = meta.loadedFromDisk && !(meta.originalLockfileVersion >= 2)

    if (inventory.size === 0 || !ancient && !old) {
      return
    }

    // if the lockfile is from node v5 or earlier, then we'll have to reload
    // all the manifests of everything we encounter.  this is costly, but at
    // least it's just a one-time hit.
    const timeEnd = time.start('idealTree:inflate')

    // don't warn if we're not gonna actually write it back anyway.
    const heading = ancient ? 'ancient lockfile' : 'old lockfile'
    if (ancient || !this.options.lockfileVersion ||
        this.options.lockfileVersion >= defaultLockfileVersion) {
      log.warn(heading,
        `
The ${meta.type} file was created with an old version of npm,
so supplemental metadata must be fetched from the registry.

This is a one-time fix-up, please be patient...
`)
    }

    this.addTracker('idealTree:inflate')
    const queue = []
    for (const node of inventory.values()) {
      if (node.isProjectRoot) {
        continue
      }

      // if the node's location isn't within node_modules then this is actually
      // a link target, so skip it. the link node itself will be queued later.
      if (!node.location.startsWith('node_modules')) {
        continue
      }

      queue.push(async () => {
        log.silly('inflate', node.location)
        const { resolved, version, path, name, location, integrity } = node
        // don't try to hit the registry for linked deps
        const useResolved = resolved && (
          !version || resolved.startsWith('file:')
        )
        const id = useResolved ? resolved : version
        const spec = npa.resolve(name, id, dirname(path))
        const t = `idealTree:inflate:${location}`
        this.addTracker(t)
        try {
          const mani = await pacote.manifest(spec, {
            ...this.options,
            resolved: resolved,
            integrity: integrity,
            fullMetadata: false,
            before: this.#releaseAgeBefore(spec),
          })
          node.package = { ...mani, _id: `${mani.name}@${mani.version}` }
        } catch (er) {
          const warning = `Could not fetch metadata for ${name}@${id}`
          log.warn(heading, warning, er)
        }
        this.finishTracker(t)
      })
    }
    await promiseCallLimit(queue)

    // have to re-calc dep flags, because the nodes don't have edges
    // until their packages get assigned, so everything looks extraneous
    resetDepFlags(this.idealTree)
    calcDepFlags(this.idealTree)

    // yes, yes, this isn't the "original" version, but now that it's been
    // upgraded, we need to make sure we don't do the work to upgrade it
    // again, since it's now as new as can be.
    if (!this.options.lockfileVersion && !meta.hiddenLockfile) {
      meta.originalLockfileVersion = defaultLockfileVersion
    }
    this.finishTracker('idealTree:inflate')
    timeEnd()
  }

  // at this point we have a virtual tree with the actual root node's
  // package deps, which may be partly or entirely incomplete, invalid
  // or extraneous.
  #buildDeps () {
    const timeEnd = time.start('idealTree:buildDeps')
    const tree = this.idealTree.target
    tree.assertRootOverrides()
    this.#depsQueue.push(tree)
    // XXX also push anything that depends on a node with a name
    // in the override list
    log.silly('idealTree', 'buildDeps')
    this.addTracker('idealTree', tree.name, '')
    return this.#buildDepStep().then(timeEnd)
  }

  async #buildDepStep () {
    // removes tracker of previous dependency in the queue
    if (this.#currentDep) {
      const { location, name } = this.#currentDep
      time.end(`idealTree:${location || '#root'}`)
      this.finishTracker('idealTree', name, location)
      this.#currentDep = null
    }

    if (!this.#depsQueue.length) {
      return this.#resolveLinks()
    }

    const node = this.#depsQueue.pop()
    const bd = node.package.bundleDependencies
    const hasBundle = bd && Array.isArray(bd) && bd.length

    // if the node was already visited, or has since been removed from the
    // tree, skip over it and process the rest of the queue.
    if (this.#depsSeen.has(node) ||
        node.root !== this.idealTree) {
      return this.#buildDepStep()
    }

    this.#depsSeen.add(node)
    this.#currentDep = node
    time.start(`idealTree:${node.location || '#root'}`)

    // if we're loading a _complete_ ideal tree, for a --package-lock-only
    // installation for example, we have to crack open the tarball and
    // look inside if it has bundle deps.  note that this is
    // not necessary during a reification, because we just update the
    // ideal tree by reading bundles in place.
    // Don't bother if the node is from the actual tree and hasn't
    // been resolved, because we can't fetch it anyway, could be anything!
    const crackOpen = this.#complete &&
      node !== this.idealTree &&
      node.resolved &&
      hasBundle &&
      !node.inert
    if (crackOpen) {
      const Arborist = this.constructor
      const opt = { ...this.options }
      await cacache.tmp.withTmp(this.cache, opt, async path => {
        await pacote.extract(node.resolved, path, {
          ...opt,
          Arborist,
          resolved: node.resolved,
          integrity: node.integrity,
        })

        await new Arborist({ ...this.options, path })
          .loadActual({ root: node, ignoreMissing: true })
      })
    }

    // if any deps are missing or invalid, then we fetch the manifest for
    // the thing we want, and build a new dep node from that.
    // Then, find the ideal placement for that node.  The ideal placement
    // searches from the node's deps (or parent deps in the case of non-root
    // peer deps), and walks up the tree until it finds the highest spot
    // where it doesn't cause any conflicts.
    //
    // A conflict can be:
    // - A node by that name already exists at that location.
    // - The parent has a peer dep on that name
    // - One of the node's peer deps conflicts at that location, unless the
    //   peer dep is met by a node at that location, which is fine.
    //
    // If we create a new node, then build its ideal deps as well.
    //
    // Note: this is the same "maximally naive" deduping tree-building
    // algorithm that npm has used since v3.  In a case like this:
    //
    // root -> (a@1, b@1||2)
    // a -> (b@1)
    //
    // You'll end up with a tree like this:
    //
    // root
    // +-- a@1
    // |   +-- b@1
    // +-- b@2
    //
    // rather than this, more deduped, but just as correct tree:
    //
    // root
    // +-- a@1
    // +-- b@1
    //
    // Another way to look at it is that this algorithm favors getting higher
    // version deps at higher levels in the tree, even if that reduces
    // potential deduplication.
    //
    // Set `preferDedupe: true` in the options to replace the shallower
    // dep if allowed.

    const tasks = []
    const peerSource = this.#peerSetSource.get(node) || node
    for (const edge of this.#problemEdges(node)) {
      if (edge.peerConflicted) {
        continue
      }

      // peerSetSource is only relevant when we have a peerEntryEdge
      // otherwise we're setting regular non-peer deps as if they have
      // a virtual root of whatever brought in THIS node.
      // so we VR the node itself if the edge is not a peer
      const source = edge.peer ? peerSource : node

      const virtualRoot = this.#virtualRoot(source, true)
      // reuse virtual root if we already have one, but don't
      // try to do the override ahead of time, since we MAY be able
      // to create a more correct tree than the virtual root could.
      const vrEdge = virtualRoot && virtualRoot.edgesOut.get(edge.name)
      const vrDep = vrEdge && vrEdge.valid && vrEdge.to
      // only re-use the virtualRoot if it's a peer edge we're placing.
      // otherwise, we end up in situations where we override peer deps that
      // we could have otherwise found homes for.  Eg:
      // xy -> (x, y)
      // x -> PEER(z@1)
      // y -> PEER(z@2)
      // If xy is a dependency, we can resolve this like:
      // project
      // +-- xy
      // |   +-- y
      // |   +-- z@2
      // +-- x
      // +-- z@1
      // But if x and y are loaded in the same virtual root, then they will
      // be forced to agree on a version of z.
      const required = new Set([edge.from])
      const parent = edge.peer ? virtualRoot : null
      let dep = vrDep && vrDep.satisfies(edge) ? vrDep : null

      // A peerOptional conflict can be resolved by finding an existing node in the tree that satisfies the edge, avoiding a registry fetch that may introduce an extraneous package. See npm/cli#9249.
      // Skip the shortcut when the user has signaled an explicit re-fetch intent (npm update by name, explicit request, or audit fix), so we honor those signals rather than silently keeping the existing node.
      const skipExistingShortcut = this[_updateNames].includes(edge.name)
        || this.#explicitRequests.has(edge)
        || (edge.to && this.auditReport?.isVulnerable(edge.to))
      if (!dep && edge.type === 'peerOptional' && !skipExistingShortcut) {
        dep = this.#findHoistableNode(
          /* istanbul ignore next - resolveParent is always set for non-root nodes */
          edge.from.resolveParent || edge.from, edge)
      }
      if (!dep) {
        dep = await this.#nodeFromEdge(edge, parent, null, required)
      }

      /* istanbul ignore next */
      debug(() => {
        if (!dep) {
          throw new Error('no dep??')
        }
      })

      tasks.push({ edge, dep })
    }

    const placeDeps = tasks.sort((a, b) => localeCompare(a.edge.name, b.edge.name))

    const promises = []
    for (const { edge, dep } of placeDeps) {
      const pd = new PlaceDep({
        edge,
        dep,

        auditReport: this.auditReport,
        explicitRequest: this.#explicitRequests.has(edge),
        force: this.options.force,
        installLinks: this.installLinks,
        installStrategy: this.#installStrategy,
        legacyPeerDeps: this.legacyPeerDeps,
        preferDedupe: this.#preferDedupe,
        strictPeerDeps: this.#strictPeerDeps,
        updateNames: this[_updateNames],
      })
      // placing a dep is actually a tree of placing the dep itself
      // and all of its peer group that aren't already met by the tree
      depth({
        tree: pd,
        getChildren: pd => pd.children,
        visit: pd => {
          const { placed, edge, canPlace: cpd, parent } = pd
          // if we didn't place anything, nothing to do here
          if (!placed) {
            return
          }

          // we placed something, that means we changed the tree
          if (placed.errors.length) {
            this.#loadFailures.add(placed)
          }
          this.#mutateTree = true
          if (cpd.canPlaceSelf === OK) {
            for (const edgeIn of placed.edgesIn) {
              if (edgeIn === edge) {
                continue
              }
              const { from, valid, peerConflicted } = edgeIn
              if (!peerConflicted && !valid) {
                if (this.#depsSeen.has(from) &&
                  (this.options.save ||
                    (this.options.save === false && this.#requestedTreeMutation))) {
                  // Re-queue already-processed nodes when a newly placed dep
                  // creates an invalid edge during npm install or another
                  // lockfile-mutating operation. This handles the case where a
                  // peerOptional dep was valid (missing) when the node was
                  // first processed, but becomes invalid when the dep is later
                  // placed by another path with a version that doesn't satisfy
                  // the peer spec.
                  // See npm/cli#8726.
                  this.#depsSeen.delete(from)
                  this.#depsQueue.push(from)
                } else if (!this.#depsSeen.has(from)) {
                  this.addTracker('idealTree', from.name, from.location)
                  this.#depsQueue.push(from)
                }
              }
            }
          } else {
            /* istanbul ignore else - should be only OK or REPLACE here */
            if (cpd.canPlaceSelf === REPLACE) {
              // this may also create some invalid edges, for example if we're
              // intentionally causing something to get nested which was
              // previously placed in this location.
              for (const edgeIn of placed.edgesIn) {
                if (edgeIn === edge) {
                  continue
                }

                const { valid, peerConflicted } = edgeIn
                if (!valid && !peerConflicted) {
                  // if it's already been visited, we have to re-visit
                  // otherwise, just enqueue normally.
                  this.#depsSeen.delete(edgeIn.from)
                  this.#depsQueue.push(edgeIn.from)
                }
              }
            }
          }

          /* istanbul ignore if - should be impossible */
          if (cpd.canPlaceSelf === CONFLICT) {
            debug(() => {
              const er = new Error('placed with canPlaceSelf=CONFLICT')
              throw Object.assign(er, { placeDep: pd })
            })
            return
          }

          // lastly, also check for the missing deps of the node we placed, and any holes created by pruning out conflicted peer sets.
          this.#depsQueue.push(placed)
          for (const dep of pd.needEvaluation) {
            this.#depsSeen.delete(dep)
            this.#depsQueue.push(dep)
          }

          // pre-fetch any problem edges, since we'll need these soon
          // if it fails at this point, though, don't worry because it may well be an optional dep that has gone missing
          // it'll fail later anyway
          for (const e of this.#problemEdges(placed)) {
            // XXX This is somehow load bearing.  This makes tests that print the ideal tree of a tree with tarball dependencies fail
            // This can't be changed or removed till we figure out why
            // The test is named "tarball deps with transitive tarball deps"
            promises.push(() =>
              this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e)), parent, e)
                .catch(() => null)
            )
          }
        },
      })
    }

    for (const { to } of node.edgesOut.values()) {
      if (to && to.isLink && to.target) {
        this.#linkNodes.add(to)
      }
    }

    await promiseCallLimit(promises)
    return this.#buildDepStep()
  }

  // BFS descendants of `root` for a node satisfying `edge`.
  // Prefers nodes closer to root.  Skips bundled nodes.
  #findHoistableNode (root, edge) {
    const queue = [...root.children.values()]
    while (queue.length) {
      const node = queue.shift()
      if (node.name === edge.name
        && !node.inDepBundle
        && node.satisfies(edge)) {
        return node
      }
      for (const child of node.children.values()) {
        queue.push(child)
      }
    }
    return null
  }

  // loads a node from an edge, and then loads its peer deps (and their peer deps, on down the line) into a virtual root parent.
  async #nodeFromEdge (edge, parent_, secondEdge, required) {
    // create a virtual root node with the same deps as the node that is requesting this one, so that we can get all the peer deps in a context where they're likely to be resolvable.
    // Note that the virtual root will also have virtual copies of the targets of any child Links, so that they resolve appropriately.
    const parent = parent_ || this.#virtualRoot(edge.from)

    const spec = npa.resolve(edge.name, edge.spec, edge.from.path)
    const first = await this.#nodeFromSpec(edge.name, spec, parent, edge)

    // we might have a case where the parent has a peer dependency on `foo@*` which resolves to v2, but another dep in the set has a peerDependency on `foo@1`.
    // In that case, if we force it to be v2, we're unnecessarily triggering an ERESOLVE.
    // If we have a second edge to worry about, and it's not satisfied by the first node, try a second and see if that satisfies the original edge here.
    const spec2 = secondEdge && npa.resolve(
      edge.name,
      secondEdge.spec,
      secondEdge.from.path
    )
    const second = secondEdge && !secondEdge.valid
      ? await this.#nodeFromSpec(edge.name, spec2, parent, secondEdge)
      : null

    // pick the second one if they're both happy with that; otherwise, first
    const node = second && edge.valid ? second : first
    // ensure the one we want is the one that's placed
    node.parent = parent

    if (required.has(edge.from) && edge.type !== 'peerOptional' ||
        secondEdge && (
          required.has(secondEdge.from) && secondEdge.type !== 'peerOptional')) {
      required.add(node)
    }

    // keep track of the thing that caused this node to be included.
    const src = parent.sourceReference
    this.#peerSetSource.set(node, src)

    // do not load the peers along with the set if this is a global top pkg
    // otherwise we'll be tempted to put peers as other top-level installed
    // things, potentially clobbering what's there already, which is not
    // what we want.  the missing edges will be picked up on the next pass.
    if (this.options.global && edge.from.isProjectRoot) {
      return node
    }

    // otherwise, we have to make sure that our peers can go along with us.
    return this.#loadPeerSet(node, required)
  }

  #virtualRoot (node, reuse = false) {
    if (reuse && this.#virtualRoots.has(node)) {
      return this.#virtualRoots.get(node)
    }

    const vr = new Node({
      path: node.realpath,
      sourceReference: node,
      installLinks: this.installLinks,
      legacyPeerDeps: this.legacyPeerDeps,
      overrides: node.overrides,
    })

    // also need to set up any targets from any link deps, so that
    // they are properly reflected in the virtual environment
    for (const child of node.children.values()) {
      if (child.isLink) {
        new Node({
          path: child.realpath,
          sourceReference: child.target,
          root: vr,
        })
      }
    }

    this.#virtualRoots.set(node, vr)
    return vr
  }

  #problemEdges (node) {
    // skip over any bundled deps, they're not our problem.
    // Note that this WILL fetch bundled meta-deps which are also dependencies
    // but not listed as bundled deps.  When reifying, we first unpack any
    // nodes that have bundleDependencies, then do a loadActual on them, move
    // the nodes into the ideal tree, and then prune.  So, fetching those
    // possibly-bundled meta-deps at this point doesn't cause any worse
    // problems than a few unnecessary packument fetches.

    // also skip over any nodes in the tree that failed to load, since those
    // will crash the install later on anyway.
    const bd = node.isProjectRoot || node.isWorkspace ? null
      : node.package.bundleDependencies
    const bundled = new Set(bd || [])

    const problems = []
    for (const edge of node.edgesOut.values()) {
      // If it's included in a bundle, we take whatever is specified.
      if (bundled.has(edge.name)) {
        continue
      }

      // If it's already been logged as a load failure, skip it.
      if (edge.to && this.#loadFailures.has(edge.to)) {
        continue
      }

      // If the edge has no destination, that's a problem, unless
      // if it's peerOptional and not explicitly requested.
      if (!edge.to) {
        if (edge.type !== 'peerOptional' ||
          this.#explicitRequests.has(edge)) {
          problems.push(edge)
        }
        continue
      }

      // If the edge has an error, there's a problem, unless it's peerOptional
      // and we're not saving or otherwise mutating (e.g. npm ci), in which
      // case we trust the lockfile and skip re-resolution. When saving (npm
      // install) or updating, peerOptional invalid edges ARE treated as
      // problems so the lockfile gets fixed.
      // See npm/cli#8726.
      if (!edge.valid) {
        if (edge.type !== 'peerOptional' ||
          this.options.save !== false ||
          this.#requestedTreeMutation) {
          problems.push(edge)
        }
        continue
      }

      // If the edge is a workspace, and it's valid, leave it alone
      if (edge.to.isWorkspace) {
        continue
      }

      // user explicitly asked to update this package by name, problem
      if (this[_updateNames].includes(edge.name)) {
        problems.push(edge)
        continue
      }

      // fixing a security vulnerability with this package, problem
      if (this.auditReport && this.auditReport.isVulnerable(edge.to)) {
        problems.push(edge)
        continue
      }

      // user has explicitly asked to install this package, problem
      if (this.#explicitRequests.has(edge)) {
        problems.push(edge)
        continue
      }
    }
    return problems
  }

  // The effective `before` filter for a package, applying `min-release-age-exclude`.
  // Returns null (no age filter) for an exempted package, otherwise the
  // configured `before`. The exemption is keyed on the spec's trusted registry
  // identity (alias targets are unwrapped) so an `npm:` alias key cannot disable
  // the filter for the package it actually resolves to.
  #releaseAgeBefore (spec) {
    const { before, minReleaseAgeExclude } = this.options
    if (!before) {
      return before
    }
    return isReleaseAgeExcluded(trustedSpecName(spec), minReleaseAgeExclude) ? null : before
  }

  async #fetchManifest (spec, parent, edge) {
    // Enforce allow-* gates before consulting the manifest cache so a cached entry from a different edge cannot bypass the policy.
    this.#checkAllow(spec, edge)
    const options = {
      ...this.options,
      avoid: this.#avoidRange(spec.name),
      fullMetadata: true,
      before: this.#releaseAgeBefore(spec),
      _isRoot: !!(edge?.from?.isProjectRoot || edge?.from?.isWorkspace),
    }
    // get the intended spec and stored metadata from yarn.lock file,
    // if available and valid.
    spec = this.idealTree.meta.checkYarnLock(spec, options)

    if (this.#manifests.has(spec.raw)) {
      return this.#manifests.get(spec.raw)
    } else {
      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, redact(spec.rawSpec)))
      const mani = await pacote.manifest(spec, options)
      this.#manifests.set(spec.raw, mani)
      return mani
    }
  }

  async #nodeFromSpec (name, spec, parent, edge) {
    // Enforce allow-git / allow-file / allow-directory / allow-remote before any branching, so the symlink (Link) path is enforced as well as the manifest-fetch path.
    // Route the failure through #loadFailures so optional-dep semantics apply (e.g. a transitive optionalDependencies entry that resolves to a disallowed git URL is silently dropped rather than failing the install).
    try {
      this.#checkAllow(spec, edge)
    } catch (error) {
      return this.#failureNode(name, parent, error, edge)
    }

    // pacote will slap integrity on its options, so we have to clone the object so it doesn't get mutated.
    // Don't bother to load the manifest for link deps, because the target might be within another package that doesn't exist yet.
    const { installLinks, legacyPeerDeps } = this
    const isWorkspace = this.idealTree.workspaces && this.idealTree.workspaces.has(spec.name)

    // spec is a directory, link it if:
    // - it's a workspace, OR
    // - it's a project-internal file: dependency (always linked), OR
    // - it's external and installLinks is false
    // TODO post arborist refactor, will need to check for installStrategy=linked
    let isProjectInternalFileSpec = false
    if (edge?.rawSpec.startsWith('file:../') || edge?.rawSpec.startsWith('file:./')) {
      const targetPath = resolve(parent.realpath, edge.rawSpec.slice(5))
      const resolvedProjectRoot = resolve(this.idealTree.realpath)
      // Check if the target is within the project root
      isProjectInternalFileSpec = targetPath.startsWith(resolvedProjectRoot + sep) || targetPath === resolvedProjectRoot
    }

    // When using --install-links, we need to handle transitive file dependencies specially
    // If the parent was installed (not linked) due to --install-links, and this is a file: dep, we should also install it rather than link it
    const parentWasInstalled = parent && !parent.isLink && parent.resolved?.startsWith('file:')
    const isTransitiveFileDep = spec.type === 'directory' && parentWasInstalled && installLinks

    // Decide whether to link or copy the dependency
    const shouldLink = (isWorkspace || isProjectInternalFileSpec || !installLinks) && !isTransitiveFileDep
    if (spec.type === 'directory' && shouldLink) {
      const realpath = spec.fetchSpec
      const { content: pkg } = await PackageJson.normalize(realpath).catch(() => {
        return { content: {} }
      })
      const link = new Link({ name, parent, realpath, pkg, installLinks, legacyPeerDeps })
      this.#linkNodes.add(link)
      return link
    }

    // if the spec matches a workspace name, then see if the workspace node will satisfy the edge. if it does, we return the workspace node to make sure it takes priority.
    if (isWorkspace) {
      const existingNode = this.idealTree.edgesOut.get(spec.name).to
      if (existingNode && existingNode.isWorkspace && existingNode.satisfies(edge)) {
        return existingNode
      }
    }

    // For file: dependencies that we're installing (not linking), ensure proper resolution
    if (isTransitiveFileDep && edge) {
      // For transitive file deps, resolve relative to the parent's original source location
      const parentOriginalPath = parent.resolved.slice(5) // Remove 'file:' prefix
      const relativePath = edge.rawSpec.slice(5) // Remove 'file:' prefix
      const absolutePath = resolve(parentOriginalPath, relativePath)
      spec = npa.resolve(name, `file:${absolutePath}`)
    }

    // spec isn't a directory, and either isn't a workspace or the workspace we have
    // doesn't satisfy the edge. try to fetch a manifest and build a node from that.
    return this.#fetchManifest(spec, parent, edge)
      .then(
        pkg => {
          // When a proxy/upstream registry returns an incomplete manifest
          // (e.g. missing version field for platform-specific packages it
          // hasn't cached), treat it as a load failure so that optional deps
          // are properly pruned instead of written to the lockfile without
          // version metadata.  Only apply to registry specs — file: deps
          // legitimately omit version.
          if (!pkg.version && spec.registry) {
            const error = Object.assign(
              new Error(`incomplete manifest for ${name}, missing version`),
              { code: 'EINCOMPLETEMANIFEST' }
            )
            return this.#failureNode(name, parent, error, edge)
          }
          // Transform the manifest copy before any packageExtensions rule and before the Node reads its dependency and peer edges.
          const transformed = this.#applyNpmExtension(pkg)
          const { pkg: extended, applied } = this.#applyPackageExtension(transformed.pkg)
          const node = new Node({ name, pkg: extended, parent, installLinks, legacyPeerDeps })
          if (transformed.applied) {
            node.npmExtensionApplied = transformed.applied
          }
          if (applied) {
            node.packageExtensionsApplied = applied
          }
          return node
        },
        error => this.#failureNode(name, parent, error, edge)
      )
  }

  // load all peer deps and meta-peer deps into the node's parent
  // At the end of this, the node's peer-type outward edges are all
  // resolved, and so are all of theirs, but other dep types are not.
  // We prefer to get peer deps that meet the requiring node's dependency,
  // if possible, since that almost certainly works (since that package was
  // developed with this set of deps) and will typically be more restrictive.
  // Note that the peers in the set can conflict either with each other,
  // or with a direct dependency from the virtual root parent!  In strict
  // mode, this is always an error.  In force mode, it never is, and we
  // prefer the parent's non-peer dep over a peer dep, or the version that
  // gets placed first.  In non-strict mode, we behave strictly if the
  // virtual root is based on the root project, and allow non-peer parent
  // deps to override, but throw if no preference can be determined.
  async #loadPeerSet (node, required) {
    const peerEdges = [...node.edgesOut.values()]
      // we typically only install non-optional peers, but we have to
      // factor them into the peerSet so that we can avoid conflicts
      .filter(e => e.peer && !(e.valid && e.to))
      .sort(({ name: a }, { name: b }) => localeCompare(a, b))

    for (const edge of peerEdges) {
      // node.parent gets mutated during loop execution due to recursive #nodeFromEdge calls.
      // When a compatible peer is found (e.g. a@1.1.0 replaces a@1.2.0), the original node loses its parent.
      // if node is detached/removed from the tree, or has no parent, so no need to check remaining edgesOut for that node.
      if (!node.parent) {
        break
      }
      // already placed this one, and we're happy with it.
      if (edge.valid && edge.to) {
        continue
      }

      const parentEdge = node.parent.edgesOut.get(edge.name)
      const { isProjectRoot, isWorkspace } = node.parent.sourceReference
      const isMine = isProjectRoot || isWorkspace
      const conflictOK = this.options.force || !isMine && !this.#strictPeerDeps

      if (!edge.to) {
        if (!parentEdge) {
          // easy, just put the thing there
          await this.#nodeFromEdge(edge, node.parent, null, required)
          continue
        } else {
          // if the parent's edge is very broad like >=1, and the edge in
          // question is something like 1.x, then we want to get a 1.x, not
          // a 2.x.  pass along the child edge as an advisory guideline.
          // if the parent edge doesn't satisfy the child edge, and the
          // child edge doesn't satisfy the parent edge, then we have
          // a conflict.  this is always a problem in strict mode, never
          // in force mode, and a problem in non-strict mode if this isn't
          // on behalf of our project.  in all such cases, we warn at least.
          const dep = await this.#nodeFromEdge(
            parentEdge,
            node.parent,
            edge,
            required
          )

          // hooray! that worked!
          if (edge.valid) {
            continue
          }

          // allow it.  either we're overriding, or it's not something
          // that will be installed by default anyway, and we'll fail when
          // we get to the point where we need to, if we need to.
          if (conflictOK || !required.has(dep)) {
            edge.peerConflicted = true
            continue
          }

          // problem
          this.#failPeerConflict(edge, parentEdge)
        }
      }

      // There is something present already, and we're not happy about it
      // See if the thing we WOULD be happy with is also going to satisfy
      // the other dependents on the current node.
      const current = edge.to
      const dep = await this.#nodeFromEdge(edge, null, null, required)
      if (dep.canReplace(current)) {
        await this.#nodeFromEdge(edge, node.parent, null, required)
        continue
      }

      // at this point we know that there is a dep there, and
      // we don't like it.  always fail strictly, always allow forcibly or
      // in non-strict mode if it's not our fault.  don't warn here, because
      // we are going to warn again when we place the deps, if we end up
      // overriding for something else.  If the thing that has this dep
      // isn't also required, then there's a good chance we won't need it,
      // so allow it for now and let it conflict if it turns out to actually
      // be necessary for the installation.
      if (conflictOK || !required.has(edge.from)) {
        continue
      }

      // ok, it's the root, or we're in unforced strict mode, so this is bad
      this.#failPeerConflict(edge, parentEdge)
    }
    return node
  }

  #failPeerConflict (edge, currentEdge) {
    const expl = this.#explainPeerConflict(edge, currentEdge)
    throw Object.assign(new Error('unable to resolve dependency tree'), expl)
  }

  #explainPeerConflict (edge, currentEdge) {
    const node = edge.from
    const curNode = node.resolve(edge.name)
    const current = curNode.explain()
    return {
      code: 'ERESOLVE',
      current,
      // it SHOULD be impossible to get here without a current node in place,
      // but this at least gives us something report on when bugs creep into
      // the tree handling logic.
      currentEdge: currentEdge ? currentEdge.explain() : null,
      edge: edge.explain(),
      strictPeerDeps: this.#strictPeerDeps,
      force: this.options.force,
    }
  }

  // go through all the links in the this.#linkNodes set
  // for each one:
  // - if outside the root, ignore it, assume it's fine, it's not our problem
  // - if a node in the tree already, assign the target to that node.
  // - if a path under an existing node, then assign that as the fsParent,
  //   and add it to the _depsQueue
  //
  // call buildDepStep if anything was added to the queue; otherwise, we're done
  // XXX load-virtual also has a #resolveLinks, is there overlap?
  #resolveLinks () {
    for (const link of this.#linkNodes) {
      this.#linkNodes.delete(link)

      // link we never ended up placing, skip it
      if (link.root !== this.idealTree) {
        continue
      }

      const tree = this.idealTree.target
      const external = !link.target.isDescendantOf(tree)

      // outside the root, somebody else's problem, ignore it
      if (external && !this.#follow) {
        continue
      }

      // didn't find a parent for it or it has not been seen yet
      // so go ahead and process it.
      const unseenLink = (link.target.parent || link.target.fsParent) &&
        !this.#depsSeen.has(link.target)

      if (this.#follow &&
          !link.target.parent &&
          !link.target.fsParent ||
          unseenLink) {
        // Forward the link's overrides before its subtree resolves, so a root override reaches a transitive dep across the link boundary (npm/cli#9659).
        if (link.overrides) {
          link.target.updateOverridesEdgeInAdded(link.overrides)
        }
        this.addTracker('idealTree', link.target.name, link.target.location)
        this.#depsQueue.push(link.target)
      }
    }

    if (this.#depsQueue.length) {
      return this.#buildDepStep()
    }
  }

  #fixDepFlags () {
    const timeEnd = time.start('idealTree:fixDepFlags')
    const metaFromDisk = this.idealTree.meta.loadedFromDisk
    const flagsSuspect = this[_flagsSuspect]
    const mutateTree = this.#mutateTree
    // if the options set prune:false, then we don't prune, but we still
    // mark the extraneous items in the tree if we modified it at all.
    // If we did no modifications, we just iterate over the extraneous nodes.
    // if we started with an empty tree, then the dep flags are already
    // all set to true, and there can be nothing extraneous, so there's
    // nothing to prune, because we built it from scratch.  if we didn't
    // add or remove anything, then also nothing to do.
    if (metaFromDisk && mutateTree) {
      resetDepFlags(this.idealTree)
    }

    // update all the dev/optional/etc flags in the tree
    // either we started with a fresh tree, or we
    // reset all the flags to find the extraneous nodes.
    //
    // if we started from a blank slate, or changed something, then
    // the dep flags will be all set to true.
    if (!metaFromDisk || mutateTree) {
      calcDepFlags(this.idealTree)
    } else {
      // otherwise just unset all the flags on the root node
      // since they will sometimes have the default value
      this.idealTree.unsetDepFlags()
    }

    // at this point, any node marked as extraneous should be pruned.
    // if we started from a shrinkwrap, and then added/removed something,
    // then the tree is suspect.  Prune what is marked as extraneous.
    // otherwise, don't bother.
    const needPrune = metaFromDisk && (mutateTree || flagsSuspect)
    if (this.#prune && needPrune) {
      this.#idealTreePrune()
    }

    timeEnd()
  }

  #idealTreePrune () {
    for (const node of this.idealTree.inventory.values()) {
      if (node.extraneous) {
        node.parent = null
      }
    }
  }

  #pruneFailedOptional () {
    for (const node of this.#loadFailures) {
      if (!node.optional) {
        throw node.errors[0]
      }

      const set = optionalSet(node)
      for (const node of set) {
        node.inert = true
      }
    }
  }

  async prune (options = {}) {
    // allow the user to set options on the ctor as well.
    // XXX: deprecate separate method options objects.
    options = { ...this.options, ...options }

    await this.buildIdealTree(options)

    this.#idealTreePrune()

    if (!this.options.workspacesEnabled) {
      const excludeNodes = this.excludeWorkspacesDependencySet(this.idealTree)
      for (const node of this.idealTree.inventory.values()) {
        if (
          node.parent !== null
          && !node.isProjectRoot
          && !excludeNodes.has(node)
          && !node.inert
        ) {
          this[_addNodeToTrashList](node)
        }
      }
    }

    return this.reify(options)
  }
}
