ReactFlow前端拖拽模型与参数配置功能MVP

This commit is contained in:
ljz committed 2026-07-11 10:49:11 +08:00
1 parent 70c91ed019
commit d3b99304fb
2001 files changed
+349044 -548

No files matched your search

@@ -0,0 +1,20 @@
Copyright (c) 2011 Tim Koschützki (tim@debuggable.com), Felix Geisendörfer (felix@debuggable.com)
Copyright (c) 2014 IndigoUnited
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@@ -0,0 +1,62 @@
const { RetryOperation } = require('./retry')
const createTimeout = (attempt, opts) => Math.min(Math.round((1 + (opts.randomize ? Math.random() : 0)) * Math.max(opts.minTimeout, 1) * Math.pow(opts.factor, attempt)), opts.maxTimeout)
const isRetryError = err => err?.code === 'EPROMISERETRY' && Object.hasOwn(err, 'retried')
const promiseRetry = async (fn, options = {}) => {
let timeouts = []
if (options instanceof Array) {
timeouts = [...options]
} else {
if (options.retries === Infinity) {
options.forever = true
delete options.retries
}
const opts = {
retries: 10,
factor: 2,
minTimeout: 1 * 1000,
maxTimeout: Infinity,
randomize: false,
...options
}
if (opts.minTimeout > opts.maxTimeout) {
throw new Error('minTimeout is greater than maxTimeout')
}
if (opts.retries) {
for (let i = 0; i < opts.retries; i++) {
timeouts.push(createTimeout(i, opts))
}
// sort the array numerically ascending (since the timeouts may be out of order at factor < 1)
timeouts.sort((a, b) => a - b)
} else if (options.forever) {
timeouts.push(createTimeout(0, opts))
}
}
const operation = new RetryOperation(timeouts, {
forever: options.forever,
unref: options.unref,
maxRetryTime: options.maxRetryTime
})
return new Promise(function (resolve, reject) {
operation.attempt(async number => {
try {
const result = await fn(err => {
throw Object.assign(new Error('Retrying'), { code: 'EPROMISERETRY', retried: err })
}, number, operation)
return resolve(result)
} catch (err) {
if (!isRetryError(err)) {
return reject(err)
}
if (!operation.retry(err.retried || new Error())) {
return reject(err.retried)
}
}
})
})
}
module.exports = { promiseRetry }
@@ -0,0 +1,109 @@
class RetryOperation {
#attempts = 1
#cachedTimeouts = null
#errors = []
#fn = null
#maxRetryTime
#operationStart = null
#originalTimeouts
#timeouts
#timer = null
#unref
constructor (timeouts, options = {}) {
this.#originalTimeouts = [...timeouts]
this.#timeouts = [...timeouts]
this.#unref = options.unref
this.#maxRetryTime = options.maxRetryTime || Infinity
if (options.forever) {
this.#cachedTimeouts = [...this.#timeouts]
}
}
get timeouts () {
return [...this.#timeouts]
}
get errors () {
return [...this.#errors]
}
get attempts () {
return this.#attempts
}
get mainError () {
let mainError = null
if (this.#errors.length) {
let mainErrorCount = 0
const counts = {}
for (let i = 0; i < this.#errors.length; i++) {
const error = this.#errors[i]
const { message } = error
if (!counts[message]) {
counts[message] = 0
}
counts[message]++
if (counts[message] >= mainErrorCount) {
mainError = error
mainErrorCount = counts[message]
}
}
}
return mainError
}
reset () {
this.#attempts = 1
this.#timeouts = [...this.#originalTimeouts]
}
stop () {
if (this.#timer) {
clearTimeout(this.#timer)
}
this.#timeouts = []
this.#cachedTimeouts = null
}
retry (err) {
this.#errors.push(err)
if (new Date().getTime() - this.#operationStart >= this.#maxRetryTime) {
// XXX This puts the timeout error first, meaning it will never show as mainError, there may be no way to ever see this
this.#errors.unshift(new Error('RetryOperation timeout occurred'))
return false
}
let timeout = this.#timeouts.shift()
if (timeout === undefined) {
// We're out of timeouts, clear the last error and repeat the final timeout
if (this.#cachedTimeouts) {
this.#errors.pop()
timeout = this.#cachedTimeouts.at(-1)
} else {
return false
}
}
// TODO what if there already is a timer?
this.#timer = setTimeout(() => {
this.#attempts++
this.#fn(this.#attempts)
}, timeout)
if (this.#unref) {
this.#timer.unref()
}
return true
}
attempt (fn) {
this.#fn = fn
this.#operationStart = new Date().getTime()
this.#fn(this.#attempts)
}
}
module.exports = { RetryOperation }
@@ -0,0 +1,45 @@
{
"name": "@gar/promise-retry",
"version": "1.0.3",
"description": "Retries a function that returns a promise, leveraging the power of the retry module.",
"main": "./lib/index.js",
"files": [
"lib"
],
"type": "commonjs",
"exports": {
".": [
{
"default": "./lib/index.js",
"types": "./lib/index.d.ts"
},
"./lib/index.js"
]
},
"scripts": {
"lint": "npx standard",
"lint:fix": "npx standard --fix",
"test": "node --test --experimental-test-coverage --test-coverage-lines=100 --test-coverage-functions=100 --test-coverage-branches=100",
"typelint": "npx -p typescript tsc ./lib/index.d.ts",
"posttest": "npm run lint",
"postlint": "npm run typelint"
},
"bugs": {
"url": "https://github.com/wraithgar/node-promise-retry/issues/"
},
"repository": {
"type": "git",
"url": "git://github.com/wraithgar/node-promise-retry.git"
},
"keywords": [
"retry",
"promise",
"backoff",
"repeat",
"replay"
],
"license": "MIT",
"engines": {
"node": "^20.17.0 || >=22.9.0"
}
}
@@ -0,0 +1,15 @@
The ISC License
Copyright (c) Isaac Z. Schlueter and Contributors
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1,430 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WriteStreamSync = exports.WriteStream = exports.ReadStreamSync = exports.ReadStream = void 0;
const events_1 = __importDefault(require("events"));
const fs_1 = __importDefault(require("fs"));
const minipass_1 = require("minipass");
const writev = fs_1.default.writev;
const _autoClose = Symbol('_autoClose');
const _close = Symbol('_close');
const _ended = Symbol('_ended');
const _fd = Symbol('_fd');
const _finished = Symbol('_finished');
const _flags = Symbol('_flags');
const _flush = Symbol('_flush');
const _handleChunk = Symbol('_handleChunk');
const _makeBuf = Symbol('_makeBuf');
const _mode = Symbol('_mode');
const _needDrain = Symbol('_needDrain');
const _onerror = Symbol('_onerror');
const _onopen = Symbol('_onopen');
const _onread = Symbol('_onread');
const _onwrite = Symbol('_onwrite');
const _open = Symbol('_open');
const _path = Symbol('_path');
const _pos = Symbol('_pos');
const _queue = Symbol('_queue');
const _read = Symbol('_read');
const _readSize = Symbol('_readSize');
const _reading = Symbol('_reading');
const _remain = Symbol('_remain');
const _size = Symbol('_size');
const _write = Symbol('_write');
const _writing = Symbol('_writing');
const _defaultFlag = Symbol('_defaultFlag');
const _errored = Symbol('_errored');
class ReadStream extends minipass_1.Minipass {
[_errored] = false;
[_fd];
[_path];
[_readSize];
[_reading] = false;
[_size];
[_remain];
[_autoClose];
constructor(path, opt) {
opt = opt || {};
super(opt);
this.readable = true;
this.writable = false;
if (typeof path !== 'string') {
throw new TypeError('path must be a string');
}
this[_errored] = false;
this[_fd] = typeof opt.fd === 'number' ? opt.fd : undefined;
this[_path] = path;
this[_readSize] = opt.readSize || 16 * 1024 * 1024;
this[_reading] = false;
this[_size] = typeof opt.size === 'number' ? opt.size : Infinity;
this[_remain] = this[_size];
this[_autoClose] =
typeof opt.autoClose === 'boolean' ? opt.autoClose : true;
if (typeof this[_fd] === 'number') {
this[_read]();
}
else {
this[_open]();
}
}
get fd() {
return this[_fd];
}
get path() {
return this[_path];
}
//@ts-ignore
write() {
throw new TypeError('this is a readable stream');
}
//@ts-ignore
end() {
throw new TypeError('this is a readable stream');
}
[_open]() {
fs_1.default.open(this[_path], 'r', (er, fd) => this[_onopen](er, fd));
}
[_onopen](er, fd) {
if (er) {
this[_onerror](er);
}
else {
this[_fd] = fd;
this.emit('open', fd);
this[_read]();
}
}
[_makeBuf]() {
return Buffer.allocUnsafe(Math.min(this[_readSize], this[_remain]));
}
[_read]() {
if (!this[_reading]) {
this[_reading] = true;
const buf = this[_makeBuf]();
/* c8 ignore start */
if (buf.length === 0) {
return process.nextTick(() => this[_onread](null, 0, buf));
}
/* c8 ignore stop */
fs_1.default.read(this[_fd], buf, 0, buf.length, null, (er, br, b) => this[_onread](er, br, b));
}
}
[_onread](er, br, buf) {
this[_reading] = false;
if (er) {
this[_onerror](er);
}
else if (this[_handleChunk](br, buf)) {
this[_read]();
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs_1.default.close(fd, er => er ? this.emit('error', er) : this.emit('close'));
}
}
[_onerror](er) {
this[_reading] = true;
this[_close]();
this.emit('error', er);
}
[_handleChunk](br, buf) {
let ret = false;
// no effect if infinite
this[_remain] -= br;
if (br > 0) {
ret = super.write(br < buf.length ? buf.subarray(0, br) : buf);
}
if (br === 0 || this[_remain] <= 0) {
ret = false;
this[_close]();
super.end();
}
return ret;
}
emit(ev, ...args) {
switch (ev) {
case 'prefinish':
case 'finish':
return false;
case 'drain':
if (typeof this[_fd] === 'number') {
this[_read]();
}
return false;
case 'error':
if (this[_errored]) {
return false;
}
this[_errored] = true;
return super.emit(ev, ...args);
default:
return super.emit(ev, ...args);
}
}
}
exports.ReadStream = ReadStream;
class ReadStreamSync extends ReadStream {
[_open]() {
let threw = true;
try {
this[_onopen](null, fs_1.default.openSync(this[_path], 'r'));
threw = false;
}
finally {
if (threw) {
this[_close]();
}
}
}
[_read]() {
let threw = true;
try {
if (!this[_reading]) {
this[_reading] = true;
do {
const buf = this[_makeBuf]();
/* c8 ignore start */
const br = buf.length === 0
? 0
: fs_1.default.readSync(this[_fd], buf, 0, buf.length, null);
/* c8 ignore stop */
if (!this[_handleChunk](br, buf)) {
break;
}
} while (true);
this[_reading] = false;
}
threw = false;
}
finally {
if (threw) {
this[_close]();
}
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs_1.default.closeSync(fd);
this.emit('close');
}
}
}
exports.ReadStreamSync = ReadStreamSync;
class WriteStream extends events_1.default {
readable = false;
writable = true;
[_errored] = false;
[_writing] = false;
[_ended] = false;
[_queue] = [];
[_needDrain] = false;
[_path];
[_mode];
[_autoClose];
[_fd];
[_defaultFlag];
[_flags];
[_finished] = false;
[_pos];
constructor(path, opt) {
opt = opt || {};
super(opt);
this[_path] = path;
this[_fd] = typeof opt.fd === 'number' ? opt.fd : undefined;
this[_mode] = opt.mode === undefined ? 0o666 : opt.mode;
this[_pos] = typeof opt.start === 'number' ? opt.start : undefined;
this[_autoClose] =
typeof opt.autoClose === 'boolean' ? opt.autoClose : true;
// truncating makes no sense when writing into the middle
const defaultFlag = this[_pos] !== undefined ? 'r+' : 'w';
this[_defaultFlag] = opt.flags === undefined;
this[_flags] = opt.flags === undefined ? defaultFlag : opt.flags;
if (this[_fd] === undefined) {
this[_open]();
}
}
emit(ev, ...args) {
if (ev === 'error') {
if (this[_errored]) {
return false;
}
this[_errored] = true;
}
return super.emit(ev, ...args);
}
get fd() {
return this[_fd];
}
get path() {
return this[_path];
}
[_onerror](er) {
this[_close]();
this[_writing] = true;
this.emit('error', er);
}
[_open]() {
fs_1.default.open(this[_path], this[_flags], this[_mode], (er, fd) => this[_onopen](er, fd));
}
[_onopen](er, fd) {
if (this[_defaultFlag] &&
this[_flags] === 'r+' &&
er &&
er.code === 'ENOENT') {
this[_flags] = 'w';
this[_open]();
}
else if (er) {
this[_onerror](er);
}
else {
this[_fd] = fd;
this.emit('open', fd);
if (!this[_writing]) {
this[_flush]();
}
}
}
end(buf, enc) {
if (buf) {
//@ts-ignore
this.write(buf, enc);
}
this[_ended] = true;
// synthetic after-write logic, where drain/finish live
if (!this[_writing] &&
!this[_queue].length &&
typeof this[_fd] === 'number') {
this[_onwrite](null, 0);
}
return this;
}
write(buf, enc) {
if (typeof buf === 'string') {
buf = Buffer.from(buf, enc);
}
if (this[_ended]) {
this.emit('error', new Error('write() after end()'));
return false;
}
if (this[_fd] === undefined || this[_writing] || this[_queue].length) {
this[_queue].push(buf);
this[_needDrain] = true;
return false;
}
this[_writing] = true;
this[_write](buf);
return true;
}
[_write](buf) {
fs_1.default.write(this[_fd], buf, 0, buf.length, this[_pos], (er, bw) => this[_onwrite](er, bw));
}
[_onwrite](er, bw) {
if (er) {
this[_onerror](er);
}
else {
if (this[_pos] !== undefined && typeof bw === 'number') {
this[_pos] += bw;
}
if (this[_queue].length) {
this[_flush]();
}
else {
this[_writing] = false;
if (this[_ended] && !this[_finished]) {
this[_finished] = true;
this[_close]();
this.emit('finish');
}
else if (this[_needDrain]) {
this[_needDrain] = false;
this.emit('drain');
}
}
}
}
[_flush]() {
if (this[_queue].length === 0) {
if (this[_ended]) {
this[_onwrite](null, 0);
}
}
else if (this[_queue].length === 1) {
this[_write](this[_queue].pop());
}
else {
const iovec = this[_queue];
this[_queue] = [];
writev(this[_fd], iovec, this[_pos], (er, bw) => this[_onwrite](er, bw));
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs_1.default.close(fd, er => er ? this.emit('error', er) : this.emit('close'));
}
}
}
exports.WriteStream = WriteStream;
class WriteStreamSync extends WriteStream {
[_open]() {
let fd;
// only wrap in a try{} block if we know we'll retry, to avoid
// the rethrow obscuring the error's source frame in most cases.
if (this[_defaultFlag] && this[_flags] === 'r+') {
try {
fd = fs_1.default.openSync(this[_path], this[_flags], this[_mode]);
}
catch (er) {
if (er?.code === 'ENOENT') {
this[_flags] = 'w';
return this[_open]();
}
else {
throw er;
}
}
}
else {
fd = fs_1.default.openSync(this[_path], this[_flags], this[_mode]);
}
this[_onopen](null, fd);
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs_1.default.closeSync(fd);
this.emit('close');
}
}
[_write](buf) {
// throw the original, but try to close if it fails
let threw = true;
try {
this[_onwrite](null, fs_1.default.writeSync(this[_fd], buf, 0, buf.length, this[_pos]));
threw = false;
}
finally {
if (threw) {
try {
this[_close]();
}
catch {
// ok error
}
}
}
}
}
exports.WriteStreamSync = WriteStreamSync;
//# sourceMappingURL=index.js.map
@@ -0,0 +1,3 @@
{
"type": "commonjs"
}
@@ -0,0 +1,420 @@
import EE from 'events';
import fs from 'fs';
import { Minipass } from 'minipass';
const writev = fs.writev;
const _autoClose = Symbol('_autoClose');
const _close = Symbol('_close');
const _ended = Symbol('_ended');
const _fd = Symbol('_fd');
const _finished = Symbol('_finished');
const _flags = Symbol('_flags');
const _flush = Symbol('_flush');
const _handleChunk = Symbol('_handleChunk');
const _makeBuf = Symbol('_makeBuf');
const _mode = Symbol('_mode');
const _needDrain = Symbol('_needDrain');
const _onerror = Symbol('_onerror');
const _onopen = Symbol('_onopen');
const _onread = Symbol('_onread');
const _onwrite = Symbol('_onwrite');
const _open = Symbol('_open');
const _path = Symbol('_path');
const _pos = Symbol('_pos');
const _queue = Symbol('_queue');
const _read = Symbol('_read');
const _readSize = Symbol('_readSize');
const _reading = Symbol('_reading');
const _remain = Symbol('_remain');
const _size = Symbol('_size');
const _write = Symbol('_write');
const _writing = Symbol('_writing');
const _defaultFlag = Symbol('_defaultFlag');
const _errored = Symbol('_errored');
export class ReadStream extends Minipass {
[_errored] = false;
[_fd];
[_path];
[_readSize];
[_reading] = false;
[_size];
[_remain];
[_autoClose];
constructor(path, opt) {
opt = opt || {};
super(opt);
this.readable = true;
this.writable = false;
if (typeof path !== 'string') {
throw new TypeError('path must be a string');
}
this[_errored] = false;
this[_fd] = typeof opt.fd === 'number' ? opt.fd : undefined;
this[_path] = path;
this[_readSize] = opt.readSize || 16 * 1024 * 1024;
this[_reading] = false;
this[_size] = typeof opt.size === 'number' ? opt.size : Infinity;
this[_remain] = this[_size];
this[_autoClose] =
typeof opt.autoClose === 'boolean' ? opt.autoClose : true;
if (typeof this[_fd] === 'number') {
this[_read]();
}
else {
this[_open]();
}
}
get fd() {
return this[_fd];
}
get path() {
return this[_path];
}
//@ts-ignore
write() {
throw new TypeError('this is a readable stream');
}
//@ts-ignore
end() {
throw new TypeError('this is a readable stream');
}
[_open]() {
fs.open(this[_path], 'r', (er, fd) => this[_onopen](er, fd));
}
[_onopen](er, fd) {
if (er) {
this[_onerror](er);
}
else {
this[_fd] = fd;
this.emit('open', fd);
this[_read]();
}
}
[_makeBuf]() {
return Buffer.allocUnsafe(Math.min(this[_readSize], this[_remain]));
}
[_read]() {
if (!this[_reading]) {
this[_reading] = true;
const buf = this[_makeBuf]();
/* c8 ignore start */
if (buf.length === 0) {
return process.nextTick(() => this[_onread](null, 0, buf));
}
/* c8 ignore stop */
fs.read(this[_fd], buf, 0, buf.length, null, (er, br, b) => this[_onread](er, br, b));
}
}
[_onread](er, br, buf) {
this[_reading] = false;
if (er) {
this[_onerror](er);
}
else if (this[_handleChunk](br, buf)) {
this[_read]();
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs.close(fd, er => er ? this.emit('error', er) : this.emit('close'));
}
}
[_onerror](er) {
this[_reading] = true;
this[_close]();
this.emit('error', er);
}
[_handleChunk](br, buf) {
let ret = false;
// no effect if infinite
this[_remain] -= br;
if (br > 0) {
ret = super.write(br < buf.length ? buf.subarray(0, br) : buf);
}
if (br === 0 || this[_remain] <= 0) {
ret = false;
this[_close]();
super.end();
}
return ret;
}
emit(ev, ...args) {
switch (ev) {
case 'prefinish':
case 'finish':
return false;
case 'drain':
if (typeof this[_fd] === 'number') {
this[_read]();
}
return false;
case 'error':
if (this[_errored]) {
return false;
}
this[_errored] = true;
return super.emit(ev, ...args);
default:
return super.emit(ev, ...args);
}
}
}
export class ReadStreamSync extends ReadStream {
[_open]() {
let threw = true;
try {
this[_onopen](null, fs.openSync(this[_path], 'r'));
threw = false;
}
finally {
if (threw) {
this[_close]();
}
}
}
[_read]() {
let threw = true;
try {
if (!this[_reading]) {
this[_reading] = true;
do {
const buf = this[_makeBuf]();
/* c8 ignore start */
const br = buf.length === 0
? 0
: fs.readSync(this[_fd], buf, 0, buf.length, null);
/* c8 ignore stop */
if (!this[_handleChunk](br, buf)) {
break;
}
} while (true);
this[_reading] = false;
}
threw = false;
}
finally {
if (threw) {
this[_close]();
}
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs.closeSync(fd);
this.emit('close');
}
}
}
export class WriteStream extends EE {
readable = false;
writable = true;
[_errored] = false;
[_writing] = false;
[_ended] = false;
[_queue] = [];
[_needDrain] = false;
[_path];
[_mode];
[_autoClose];
[_fd];
[_defaultFlag];
[_flags];
[_finished] = false;
[_pos];
constructor(path, opt) {
opt = opt || {};
super(opt);
this[_path] = path;
this[_fd] = typeof opt.fd === 'number' ? opt.fd : undefined;
this[_mode] = opt.mode === undefined ? 0o666 : opt.mode;
this[_pos] = typeof opt.start === 'number' ? opt.start : undefined;
this[_autoClose] =
typeof opt.autoClose === 'boolean' ? opt.autoClose : true;
// truncating makes no sense when writing into the middle
const defaultFlag = this[_pos] !== undefined ? 'r+' : 'w';
this[_defaultFlag] = opt.flags === undefined;
this[_flags] = opt.flags === undefined ? defaultFlag : opt.flags;
if (this[_fd] === undefined) {
this[_open]();
}
}
emit(ev, ...args) {
if (ev === 'error') {
if (this[_errored]) {
return false;
}
this[_errored] = true;
}
return super.emit(ev, ...args);
}
get fd() {
return this[_fd];
}
get path() {
return this[_path];
}
[_onerror](er) {
this[_close]();
this[_writing] = true;
this.emit('error', er);
}
[_open]() {
fs.open(this[_path], this[_flags], this[_mode], (er, fd) => this[_onopen](er, fd));
}
[_onopen](er, fd) {
if (this[_defaultFlag] &&
this[_flags] === 'r+' &&
er &&
er.code === 'ENOENT') {
this[_flags] = 'w';
this[_open]();
}
else if (er) {
this[_onerror](er);
}
else {
this[_fd] = fd;
this.emit('open', fd);
if (!this[_writing]) {
this[_flush]();
}
}
}
end(buf, enc) {
if (buf) {
//@ts-ignore
this.write(buf, enc);
}
this[_ended] = true;
// synthetic after-write logic, where drain/finish live
if (!this[_writing] &&
!this[_queue].length &&
typeof this[_fd] === 'number') {
this[_onwrite](null, 0);
}
return this;
}
write(buf, enc) {
if (typeof buf === 'string') {
buf = Buffer.from(buf, enc);
}
if (this[_ended]) {
this.emit('error', new Error('write() after end()'));
return false;
}
if (this[_fd] === undefined || this[_writing] || this[_queue].length) {
this[_queue].push(buf);
this[_needDrain] = true;
return false;
}
this[_writing] = true;
this[_write](buf);
return true;
}
[_write](buf) {
fs.write(this[_fd], buf, 0, buf.length, this[_pos], (er, bw) => this[_onwrite](er, bw));
}
[_onwrite](er, bw) {
if (er) {
this[_onerror](er);
}
else {
if (this[_pos] !== undefined && typeof bw === 'number') {
this[_pos] += bw;
}
if (this[_queue].length) {
this[_flush]();
}
else {
this[_writing] = false;
if (this[_ended] && !this[_finished]) {
this[_finished] = true;
this[_close]();
this.emit('finish');
}
else if (this[_needDrain]) {
this[_needDrain] = false;
this.emit('drain');
}
}
}
}
[_flush]() {
if (this[_queue].length === 0) {
if (this[_ended]) {
this[_onwrite](null, 0);
}
}
else if (this[_queue].length === 1) {
this[_write](this[_queue].pop());
}
else {
const iovec = this[_queue];
this[_queue] = [];
writev(this[_fd], iovec, this[_pos], (er, bw) => this[_onwrite](er, bw));
}
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs.close(fd, er => er ? this.emit('error', er) : this.emit('close'));
}
}
}
export class WriteStreamSync extends WriteStream {
[_open]() {
let fd;
// only wrap in a try{} block if we know we'll retry, to avoid
// the rethrow obscuring the error's source frame in most cases.
if (this[_defaultFlag] && this[_flags] === 'r+') {
try {
fd = fs.openSync(this[_path], this[_flags], this[_mode]);
}
catch (er) {
if (er?.code === 'ENOENT') {
this[_flags] = 'w';
return this[_open]();
}
else {
throw er;
}
}
}
else {
fd = fs.openSync(this[_path], this[_flags], this[_mode]);
}
this[_onopen](null, fd);
}
[_close]() {
if (this[_autoClose] && typeof this[_fd] === 'number') {
const fd = this[_fd];
this[_fd] = undefined;
fs.closeSync(fd);
this.emit('close');
}
}
[_write](buf) {
// throw the original, but try to close if it fails
let threw = true;
try {
this[_onwrite](null, fs.writeSync(this[_fd], buf, 0, buf.length, this[_pos]));
threw = false;
}
finally {
if (threw) {
try {
this[_close]();
}
catch {
// ok error
}
}
}
}
}
//# sourceMappingURL=index.js.map
@@ -0,0 +1,3 @@
{
"type": "module"
}
@@ -0,0 +1,72 @@
{
"name": "@isaacs/fs-minipass",
"version": "4.0.1",
"main": "./dist/commonjs/index.js",
"scripts": {
"prepare": "tshy",
"pretest": "npm run prepare",
"test": "tap",
"preversion": "npm test",
"postversion": "npm publish",
"prepublishOnly": "git push origin --follow-tags",
"format": "prettier --write . --loglevel warn",
"typedoc": "typedoc --tsconfig .tshy/esm.json ./src/*.ts"
},
"keywords": [],
"author": "Isaac Z. Schlueter",
"license": "ISC",
"repository": {
"type": "git",
"url": "https://github.com/npm/fs-minipass.git"
},
"description": "fs read and write streams based on minipass",
"dependencies": {
"minipass": "^7.0.4"
},
"devDependencies": {
"@types/node": "^20.11.30",
"mutate-fs": "^2.1.1",
"prettier": "^3.2.5",
"tap": "^18.7.1",
"tshy": "^1.12.0",
"typedoc": "^0.25.12"
},
"files": [
"dist"
],
"engines": {
"node": ">=18.0.0"
},
"tshy": {
"exports": {
"./package.json": "./package.json",
".": "./src/index.ts"
}
},
"exports": {
"./package.json": "./package.json",
".": {
"import": {
"types": "./dist/esm/index.d.ts",
"default": "./dist/esm/index.js"
},
"require": {
"types": "./dist/commonjs/index.d.ts",
"default": "./dist/commonjs/index.js"
}
}
},
"types": "./dist/commonjs/index.d.ts",
"type": "module",
"prettier": {
"semi": false,
"printWidth": 75,
"tabWidth": 2,
"useTabs": false,
"singleQuote": true,
"jsxSingleQuote": false,
"bracketSameLine": true,
"arrowParens": "avoid",
"endOfLine": "lf"
}
}
@@ -0,0 +1,15 @@
The ISC License
Copyright (c) Isaac Z. Schlueter
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1,42 @@
const hasIntl = typeof Intl === 'object' && !!Intl
const Collator = hasIntl && Intl.Collator
const cache = new Map()
const collatorCompare = (locale, opts) => {
const collator = new Collator(locale, opts)
return (a, b) => collator.compare(a, b)
}
const localeCompare = (locale, opts) => (a, b) => a.localeCompare(b, locale, opts)
const knownOptions = [
'sensitivity',
'numeric',
'ignorePunctuation',
'caseFirst',
]
const { hasOwnProperty } = Object.prototype
module.exports = (locale, options = {}) => {
if (!locale || typeof locale !== 'string')
throw new TypeError('locale required')
const opts = knownOptions.reduce((opts, k) => {
if (hasOwnProperty.call(options, k)) {
opts[k] = options[k]
}
return opts
}, {})
const key = `${locale}\n${JSON.stringify(opts)}`
if (cache.has(key))
return cache.get(key)
const compare = hasIntl
? collatorCompare(locale, opts)
: localeCompare(locale, opts)
cache.set(key, compare)
return compare
}
@@ -0,0 +1,28 @@
{
"name": "@isaacs/string-locale-compare",
"version": "1.1.0",
"files": [
"index.js"
],
"main": "index.js",
"description": "Compare strings with Intl.Collator if available, falling back to String.localeCompare otherwise",
"repository": {
"type": "git",
"url": "git+https://github.com/isaacs/string-locale-compare"
},
"author": "Isaac Z. Schlueter <i@izs.me> (https://izs.me)",
"license": "ISC",
"scripts": {
"test": "tap",
"snap": "tap",
"preversion": "npm test",
"postversion": "npm publish",
"prepublishOnly": "git push origin --follow-tags"
},
"tap": {
"check-coverage": true
},
"devDependencies": {
"tap": "^15.0.9"
}
}
@@ -0,0 +1,258 @@
'use strict'
const net = require('net')
const tls = require('tls')
const { once } = require('events')
const timers = require('timers/promises')
const { normalizeOptions, cacheOptions } = require('./options')
const { getProxy, getProxyAgent, proxyCache } = require('./proxy.js')
const Errors = require('./errors.js')
const { Agent: AgentBase } = require('agent-base')
module.exports = class Agent extends AgentBase {
#options
#timeouts
#proxy
#noProxy
#ProxyAgent
constructor (options = {}) {
const { timeouts, proxy, noProxy, ...normalizedOptions } = normalizeOptions(options)
super(normalizedOptions)
this.#options = normalizedOptions
this.#timeouts = timeouts
if (proxy) {
this.#proxy = new URL(proxy)
this.#noProxy = noProxy
this.#ProxyAgent = getProxyAgent(proxy)
}
}
get proxy () {
return this.#proxy ? { url: this.#proxy } : {}
}
#getProxy (options) {
if (!this.#proxy) {
return
}
const proxy = getProxy(`${options.protocol}//${options.host}:${options.port}`, {
proxy: this.#proxy,
noProxy: this.#noProxy,
})
if (!proxy) {
return
}
const cacheKey = cacheOptions({
...options,
...this.#options,
timeouts: this.#timeouts,
proxy,
})
if (proxyCache.has(cacheKey)) {
return proxyCache.get(cacheKey)
}
let ProxyAgent = this.#ProxyAgent
if (Array.isArray(ProxyAgent)) {
ProxyAgent = this.isSecureEndpoint(options) ? ProxyAgent[1] : ProxyAgent[0]
}
const proxyAgent = new ProxyAgent(proxy, {
...this.#options,
socketOptions: { family: this.#options.family },
})
proxyCache.set(cacheKey, proxyAgent)
return proxyAgent
}
// takes an array of promises and races them against the connection timeout
// which will throw the necessary error if it is hit. This will return the
// result of the promise race.
async #timeoutConnection ({ promises, options, timeout }, ac = new AbortController()) {
if (timeout) {
const connectionTimeout = timers.setTimeout(timeout, null, { signal: ac.signal })
.then(() => {
throw new Errors.ConnectionTimeoutError(`${options.host}:${options.port}`)
}).catch((err) => {
if (err.name === 'AbortError') {
return
}
throw err
})
promises.push(connectionTimeout)
}
let result
try {
result = await Promise.race(promises)
ac.abort()
} catch (err) {
ac.abort()
throw err
}
return result
}
async connect (request, options) {
// if the connection does not have its own lookup function
// set, then use the one from our options
options.lookup ??= this.#options.lookup
let socket
let timeout = this.#timeouts.connection
const isSecureEndpoint = this.isSecureEndpoint(options)
const proxy = this.#getProxy(options)
if (proxy) {
// some of the proxies will wait for the socket to fully connect before
// returning so we have to await this while also racing it against the
// connection timeout.
const start = Date.now()
socket = await this.#timeoutConnection({
options,
timeout,
promises: [proxy.connect(request, options)],
})
// see how much time proxy.connect took and subtract it from
// the timeout
if (timeout) {
timeout = timeout - (Date.now() - start)
}
} else {
socket = (isSecureEndpoint ? tls : net).connect(options)
}
socket.setKeepAlive(this.keepAlive, this.keepAliveMsecs)
socket.setNoDelay(this.keepAlive)
const abortController = new AbortController()
const { signal } = abortController
const connectPromise = socket[isSecureEndpoint ? 'secureConnecting' : 'connecting']
? once(socket, isSecureEndpoint ? 'secureConnect' : 'connect', { signal })
: Promise.resolve()
await this.#timeoutConnection({
options,
timeout,
promises: [
connectPromise,
once(socket, 'error', { signal }).then((err) => {
throw err[0]
}),
],
}, abortController)
if (this.#timeouts.idle) {
socket.setTimeout(this.#timeouts.idle, () => {
socket.destroy(new Errors.IdleTimeoutError(`${options.host}:${options.port}`))
})
}
return socket
}
addRequest (request, options) {
const proxy = this.#getProxy(options)
// it would be better to call proxy.addRequest here but this causes the
// http-proxy-agent to call its super.addRequest which causes the request
// to be added to the agent twice. since we only support 3 agents
// currently (see the required agents in proxy.js) we have manually
// checked that the only public methods we need to call are called in the
// next block. this could change in the future and presumably we would get
// failing tests until we have properly called the necessary methods on
// each of our proxy agents
if (proxy?.setRequestProps) {
proxy.setRequestProps(request, options)
}
request.setHeader('connection', this.keepAlive ? 'keep-alive' : 'close')
if (this.#timeouts.response) {
let responseTimeout
request.once('finish', () => {
setTimeout(() => {
request.destroy(new Errors.ResponseTimeoutError(request, this.#proxy))
}, this.#timeouts.response)
})
request.once('response', () => {
clearTimeout(responseTimeout)
})
}
if (this.#timeouts.transfer) {
let transferTimeout
request.once('response', (res) => {
setTimeout(() => {
res.destroy(new Errors.TransferTimeoutError(request, this.#proxy))
}, this.#timeouts.transfer)
res.once('close', () => {
clearTimeout(transferTimeout)
})
})
}
return super.addRequest(request, options)
}
// When connect() rejects, agent-base removes only its placeholder socket, so Node never drains this.requests[name] and requests queued past maxSockets hang forever.
// On a failure we dispatch the next queued request ourselves.
// See npm/cli#9386 and TooTallNate/proxy-agents#427.
createSocket (req, options, cb) {
super.createSocket(req, options, (err, socket) => {
if (err) {
this.#drainPendingRequests(req, options)
}
cb(err, socket)
})
}
// Dispatch the next request queued behind maxSockets, reusing the slot the failed connection freed.
#drainPendingRequests (failedReq, options) {
const name = this.getName(options)
const queue = this.requests[name]
if (!queue || queue.length === 0) {
return
}
// Node's removeSocket() picks a queued request without shifting it off, so drop the failed one to avoid dispatching it twice.
const failedIndex = queue.indexOf(failedReq)
if (failedIndex !== -1) {
queue.splice(failedIndex, 1)
}
if (queue.length === 0) {
delete this.requests[name]
return
}
// Safety belt: only dispatch if a socket slot is genuinely free.
const socketCount = this.sockets[name] ? this.sockets[name].length : 0
if (socketCount >= this.maxSockets || this.totalSocketCount >= this.maxTotalSockets) {
return
}
const nextReq = queue.shift()
if (queue.length === 0) {
delete this.requests[name]
}
// All queued requests share this origin, so the failed request's options suit the next one.
// createSocket() recurses here if this connection also fails, draining the whole queue.
this.createSocket(nextReq, options, (err, socket) => {
if (err) {
nextReq.onSocket(null, err)
} else {
nextReq.onSocket(socket)
}
})
}
}
@@ -0,0 +1,53 @@
'use strict'
const { LRUCache } = require('lru-cache')
const dns = require('dns')
// this is a factory so that each request can have its own opts (i.e. ttl)
// while still sharing the cache across all requests
const cache = new LRUCache({ max: 50 })
const getOptions = ({
family = 0,
hints = dns.ADDRCONFIG,
all = false,
verbatim = undefined,
ttl = 5 * 60 * 1000,
lookup = dns.lookup,
}) => ({
// hints and lookup are returned since both are top level properties to (net|tls).connect
hints,
lookup: (hostname, ...args) => {
const callback = args.pop() // callback is always last arg
const lookupOptions = args[0] ?? {}
const options = {
family,
hints,
all,
verbatim,
...(typeof lookupOptions === 'number' ? { family: lookupOptions } : lookupOptions),
}
const key = JSON.stringify({ hostname, ...options })
if (cache.has(key)) {
const cached = cache.get(key)
return process.nextTick(callback, null, ...cached)
}
lookup(hostname, options, (err, ...result) => {
if (err) {
return callback(err)
}
cache.set(key, result, { ttl })
return callback(null, ...result)
})
},
})
module.exports = {
cache,
getOptions,
}
@@ -0,0 +1,61 @@
'use strict'
class InvalidProxyProtocolError extends Error {
constructor (url) {
super(`Invalid protocol \`${url.protocol}\` connecting to proxy \`${url.host}\``)
this.code = 'EINVALIDPROXY'
this.proxy = url
}
}
class ConnectionTimeoutError extends Error {
constructor (host) {
super(`Timeout connecting to host \`${host}\``)
this.code = 'ECONNECTIONTIMEOUT'
this.host = host
}
}
class IdleTimeoutError extends Error {
constructor (host) {
super(`Idle timeout reached for host \`${host}\``)
this.code = 'EIDLETIMEOUT'
this.host = host
}
}
class ResponseTimeoutError extends Error {
constructor (request, proxy) {
let msg = 'Response timeout '
if (proxy) {
msg += `from proxy \`${proxy.host}\` `
}
msg += `connecting to host \`${request.host}\``
super(msg)
this.code = 'ERESPONSETIMEOUT'
this.proxy = proxy
this.request = request
}
}
class TransferTimeoutError extends Error {
constructor (request, proxy) {
let msg = 'Transfer timeout '
if (proxy) {
msg += `from proxy \`${proxy.host}\` `
}
msg += `for \`${request.host}\``
super(msg)
this.code = 'ETRANSFERTIMEOUT'
this.proxy = proxy
this.request = request
}
}
module.exports = {
InvalidProxyProtocolError,
ConnectionTimeoutError,
IdleTimeoutError,
ResponseTimeoutError,
TransferTimeoutError,
}
@@ -0,0 +1,56 @@
'use strict'
const { LRUCache } = require('lru-cache')
const { normalizeOptions, cacheOptions } = require('./options')
const { getProxy, proxyCache } = require('./proxy.js')
const dns = require('./dns.js')
const Agent = require('./agents.js')
const agentCache = new LRUCache({ max: 20 })
const getAgent = (url, { agent, proxy, noProxy, ...options } = {}) => {
// false has meaning so this can't be a simple truthiness check
if (agent != null) {
return agent
}
url = new URL(url)
const proxyForUrl = getProxy(url, { proxy, noProxy })
const normalizedOptions = {
...normalizeOptions(options),
proxy: proxyForUrl,
}
const cacheKey = cacheOptions({
...normalizedOptions,
secureEndpoint: url.protocol === 'https:',
})
if (agentCache.has(cacheKey)) {
return agentCache.get(cacheKey)
}
const newAgent = new Agent(normalizedOptions)
agentCache.set(cacheKey, newAgent)
return newAgent
}
module.exports = {
getAgent,
Agent,
// these are exported for backwards compatability
HttpAgent: Agent,
HttpsAgent: Agent,
cache: {
proxy: proxyCache,
agent: agentCache,
dns: dns.cache,
clear: () => {
proxyCache.clear()
agentCache.clear()
dns.cache.clear()
},
},
}
@@ -0,0 +1,90 @@
'use strict'
const dns = require('./dns')
const normalizeOptions = (opts) => {
const family = parseInt(opts.family ?? '0', 10)
const keepAlive = opts.keepAlive ?? true
const normalized = {
// nodejs http agent options. these are all the defaults
// but kept here to increase the likelihood of cache hits
// https://nodejs.org/api/http.html#new-agentoptions
keepAliveMsecs: keepAlive ? 1000 : undefined,
maxSockets: opts.maxSockets ?? 15,
maxTotalSockets: Infinity,
maxFreeSockets: keepAlive ? 256 : undefined,
scheduling: 'fifo',
// then spread the rest of the options
...opts,
// we already set these to their defaults that we want
family,
keepAlive,
// our custom timeout options
timeouts: {
// the standard timeout option is mapped to our idle timeout
// and then deleted below
idle: opts.timeout ?? 0,
connection: 0,
response: 0,
transfer: 0,
...opts.timeouts,
},
// get the dns options that go at the top level of socket connection
...dns.getOptions({ family, ...opts.dns }),
}
// remove timeout since we already used it to set our own idle timeout
delete normalized.timeout
// since opts is often passed when initiating requests, it may contain
// headers, which should not be saved in an agent
delete normalized.headers
return normalized
}
const createKey = (obj) => {
let key = ''
const sorted = Object.entries(obj).sort((a, b) => a[0] - b[0])
for (let [k, v] of sorted) {
if (v == null) {
v = 'null'
} else if (v instanceof URL) {
v = v.toString()
} else if (typeof v === 'object') {
v = createKey(v)
}
key += `${k}:${v}:`
}
return key
}
const cacheOptions = ({ secureEndpoint, ...options }) => createKey({
secureEndpoint: !!secureEndpoint,
// socket connect options
family: options.family,
hints: options.hints,
localAddress: options.localAddress,
// tls specific connect options
strictSsl: secureEndpoint ? !!options.rejectUnauthorized : false,
ca: secureEndpoint ? options.ca : null,
cert: secureEndpoint ? options.cert : null,
key: secureEndpoint ? options.key : null,
// http agent options
keepAlive: options.keepAlive,
keepAliveMsecs: options.keepAliveMsecs,
maxSockets: options.maxSockets,
maxTotalSockets: options.maxTotalSockets,
maxFreeSockets: options.maxFreeSockets,
scheduling: options.scheduling,
// timeout options
timeouts: options.timeouts,
// proxy
proxy: options.proxy,
})
module.exports = {
normalizeOptions,
cacheOptions,
}
@@ -0,0 +1,88 @@
'use strict'
const { HttpProxyAgent } = require('http-proxy-agent')
const { HttpsProxyAgent } = require('https-proxy-agent')
const { SocksProxyAgent } = require('socks-proxy-agent')
const { LRUCache } = require('lru-cache')
const { InvalidProxyProtocolError } = require('./errors.js')
const PROXY_CACHE = new LRUCache({ max: 20 })
const SOCKS_PROTOCOLS = new Set(SocksProxyAgent.protocols)
const PROXY_ENV_KEYS = new Set(['https_proxy', 'http_proxy', 'proxy', 'no_proxy'])
const PROXY_ENV = Object.entries(process.env).reduce((acc, [key, value]) => {
key = key.toLowerCase()
if (PROXY_ENV_KEYS.has(key)) {
acc[key] = value
}
return acc
}, {})
const getProxyAgent = (url) => {
url = new URL(url)
const protocol = url.protocol.slice(0, -1)
if (SOCKS_PROTOCOLS.has(protocol)) {
return SocksProxyAgent
}
if (protocol === 'https' || protocol === 'http') {
return [HttpProxyAgent, HttpsProxyAgent]
}
throw new InvalidProxyProtocolError(url)
}
const isNoProxy = (url, noProxy) => {
if (typeof noProxy === 'string') {
noProxy = noProxy.split(',').map((p) => p.trim()).filter(Boolean)
}
if (!noProxy || !noProxy.length) {
return false
}
const hostSegments = url.hostname.split('.').reverse()
return noProxy.some((no) => {
const noSegments = no.split('.').filter(Boolean).reverse()
if (!noSegments.length) {
return false
}
for (let i = 0; i < noSegments.length; i++) {
if (hostSegments[i] !== noSegments[i]) {
return false
}
}
return true
})
}
const getProxy = (url, { proxy, noProxy }) => {
url = new URL(url)
if (!proxy) {
proxy = url.protocol === 'https:'
? PROXY_ENV.https_proxy
: PROXY_ENV.https_proxy || PROXY_ENV.http_proxy || PROXY_ENV.proxy
}
if (!noProxy) {
noProxy = PROXY_ENV.no_proxy
}
if (!proxy || isNoProxy(url, noProxy)) {
return null
}
return new URL(proxy)
}
module.exports = {
getProxyAgent,
getProxy,
proxyCache: PROXY_CACHE,
}
@@ -0,0 +1,62 @@
{
"name": "@npmcli/agent",
"version": "4.0.2",
"description": "the http/https agent used by the npm cli",
"main": "lib/index.js",
"scripts": {
"gencerts": "bash scripts/create-cert.sh",
"test": "tap",
"lint": "npm run eslint",
"postlint": "template-oss-check",
"template-oss-apply": "template-oss-apply --force",
"lintfix": "npm run eslint -- --fix",
"snap": "tap",
"posttest": "npm run lint",
"eslint": "eslint \"**/*.{js,cjs,ts,mjs,jsx,tsx}\""
},
"author": "GitHub Inc.",
"license": "ISC",
"bugs": {
"url": "https://github.com/npm/agent/issues"
},
"homepage": "https://github.com/npm/agent#readme",
"files": [
"bin/",
"lib/"
],
"engines": {
"node": "^20.17.0 || >=22.9.0"
},
"templateOSS": {
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
"version": "4.30.0",
"publish": "true",
"updateNpm": false,
"latestCiVersion": 24
},
"dependencies": {
"agent-base": "^7.1.0",
"http-proxy-agent": "^7.0.0",
"https-proxy-agent": "^7.0.1",
"lru-cache": "^11.2.1",
"socks-proxy-agent": "^8.0.3"
},
"devDependencies": {
"@npmcli/eslint-config": "^6.0.0",
"@npmcli/template-oss": "4.30.0",
"ip-address": "^10.1.0",
"minipass-fetch": "^5.0.0",
"nock": "^14.0.3",
"tap": "^16.3.0"
},
"repository": {
"type": "git",
"url": "git+https://github.com/npm/agent.git"
},
"tap": {
"nyc-arg": [
"--exclude",
"tap-snapshots/**"
]
}
}
@@ -0,0 +1,20 @@
<!-- This file is automatically added by @npmcli/template-oss. Do not edit. -->
ISC License
Copyright npm, Inc.
Permission to use, copy, modify, and/or distribute this
software for any purpose with or without fee is hereby
granted, provided that the above copyright notice and this
permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND NPM DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
EVENT SHALL NPM BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1,315 @@
# @npmcli/arborist
[![npm version](https://img.shields.io/npm/v/@npmcli/arborist.svg)](https://npm.im/@npmcli/arborist)
[![license](https://img.shields.io/npm/l/@npmcli/arborist.svg)](https://npm.im/@npmcli/arborist)
[![CI - @npmcli/arborist](https://github.com/npm/cli/actions/workflows/ci-npmcli-arborist.yml/badge.svg)](https://github.com/npm/cli/actions/workflows/ci-npmcli-arborist.yml)
Inspect and manage `node_modules` trees.
![a tree with the word ARBORIST superimposed on it](https://raw.githubusercontent.com/npm/cli/latest/workspaces/arborist/docs/logo.svg?sanitize=true)
There's more documentation [in the docs
folder](https://github.com/npm/cli/tree/latest/workspaces/arborist/docs).
## USAGE
```js
const Arborist = require('@npmcli/arborist')
const arb = new Arborist({
// options object
// where we're doing stuff. defaults to cwd.
path: '/path/to/package/root',
// url to the default registry. defaults to npm's default registry
registry: 'https://registry.npmjs.org',
// scopes can be mapped to a different registry
'@foo:registry': 'https://registry.foo.com/',
// Auth can be provided in a couple of different ways. If none are
// provided, then requests are anonymous, and private packages will 404.
// Arborist doesn't do anything with these, it just passes them down
// the chain to pacote and npm-registry-fetch.
// Safest: a bearer token provided by a registry:
// 1. an npm auth token, used with the default registry
token: 'deadbeefcafebad',
// 2. an alias for the same thing:
_authToken: 'deadbeefcafebad',
// insecure options:
// 3. basic auth, username:password, base64 encoded
auth: 'aXNhYWNzOm5vdCBteSByZWFsIHBhc3N3b3Jk',
// 4. username and base64 encoded password
username: 'isaacs',
password: 'bm90IG15IHJlYWwgcGFzc3dvcmQ=',
// auth configs can also be scoped to a given registry with this
// rather unusual pattern:
'//registry.foo.com:token': 'blahblahblah',
'//basic.auth.only.foo.com:_auth': 'aXNhYWNzOm5vdCBteSByZWFsIHBhc3N3b3Jk',
'//registry.foo.com:always-auth': true,
})
// READING
// returns a promise. reads the actual contents of node_modules
arb.loadActual().then(tree => {
// tree is also stored at arb.virtualTree
})
// read just what the package-lock.json/npm-shrinkwrap says
// This *also* loads the yarn.lock file, but that's only relevant
// when building the ideal tree.
arb.loadVirtual().then(tree => {
// tree is also stored at arb.virtualTree
// now arb.virtualTree is loaded
// this fails if there's no package-lock.json or package.json in the folder
// note that loading this way should only be done if there's no
// node_modules folder
})
// OPTIMIZING AND DESIGNING
// build an ideal tree from the package.json and various lockfiles.
arb.buildIdealTree(options).then(() => {
// next step is to reify that ideal tree onto disk.
// options can be:
// rm: array of package names to remove at top level
// add: Array of package specifiers to add at the top level. Each of
// these will be resolved with pacote.manifest if the name can't be
// determined from the spec. (Eg, `github:foo/bar` vs `foo@somespec`.)
// The dep will be saved in the location where it already exists,
// (or pkg.dependencies) unless a different saveType is specified.
// saveType: Save added packages in a specific dependency set.
// - null (default) Wherever they exist already, or 'dependencies'
// - prod: definitely in 'dependencies'
// - optional: in 'optionalDependencies'
// - dev: devDependencies
// - peer: save in peerDependencies, and remove any optional flag from
// peerDependenciesMeta if one exists
// - peerOptional: save in peerDependencies, and add a
// peerDepsMeta[name].optional flag
// saveBundle: add newly added deps to the bundleDependencies list
// update: Either `true` to just go ahead and update everything, or an
// object with any or all of the following fields:
// - all: boolean. set to true to just update everything
// - names: names of packages update (like `npm update foo`)
// prune: boolean, default true. Prune extraneous nodes from the tree.
// preferDedupe: prefer to deduplicate packages if possible, rather than
// choosing a newer version of a dependency. Defaults to false, ie,
// always try to get the latest and greatest deps.
// legacyBundling: Nest every dep under the node requiring it, npm v2 style.
// No unnecessary deduplication. Default false.
// At the end of this process, arb.idealTree is set.
})
// WRITING
// Make the idealTree be the thing that's on disk
arb.reify({
// write the lockfile(s) back to disk, and package.json with any updates
// defaults to 'true'
save: true,
}).then(() => {
// node modules has been written to match the idealTree
})
```
## DATA STRUCTURES
A `node_modules` tree is a logical graph of dependencies overlaid on a
physical tree of folders.
A `Node` represents a package folder on disk, either at the root of the
package, or within a `node_modules` folder. The physical structure of the
folder tree is represented by the `node.parent` reference to the containing
folder, and `node.children` map of nodes within its `node_modules`
folder, where the key in the map is the name of the folder in
`node_modules`, and the value is the child node.
A node without a parent is a top of tree.
A `Link` represents a symbolic link to a package on disk. This can be a
symbolic link to a package folder within the current tree, or elsewhere on
disk. The `link.target` is a reference to the actual node. Links differ
from Nodes in that dependencies are resolved from the _target_ location,
rather than from the link location.
An `Edge` represents a dependency relationship. Each node has an `edgesIn`
set, and an `edgesOut` map. Each edge has a `type` which specifies what
kind of dependency it represents: `'prod'` for regular dependencies,
`'peer'` for peerDependencies, `'dev'` for devDependencies, and
`'optional'` for optionalDependencies. `edge.from` is a reference to the
node that has the dependency, and `edge.to` is a reference to the node that
requires the dependency.
As nodes are moved around in the tree, the graph edges are automatically
updated to point at the new module resolution targets. In other words,
`edge.from`, `edge.name`, and `edge.spec` are immutable; `edge.to` is
updated automatically when a node's parent changes.
### class Node
All arborist trees are `Node` objects. A `Node` refers
to a package folder, which may have children in `node_modules`.
* `node.name` The name of this node's folder in `node_modules`.
* `node.parent` Physical parent node in the tree. The package in whose
`node_modules` folder this package lives. Null if node is top of tree.
Setting `node.parent` will automatically update `node.location` and all
graph edges affected by the move.
* `node.meta` A `Shrinkwrap` object which looks up `resolved` and
`integrity` values for all modules in this tree. Only relevant on `root`
nodes.
* `node.children` Map of packages located in the node's `node_modules`
folder.
* `node.package` The contents of this node's `package.json` file.
* `node.path` File path to this package. If the node is a link, then this
is the path to the link, not to the link target. If the node is _not_ a
link, then this matches `node.realpath`.
* `node.realpath` The full real filepath on disk where this node lives.
* `node.location` A slash-normalized relative path from the root node to
this node's path.
* `node.isLink` Whether this represents a symlink. Always `false` for Node
objects, always `true` for Link objects.
* `node.isRoot` True if this node is a root node. (Ie, if `node.root ===
node`.)
* `node.root` The root node where we are working. If not assigned to some
other value, resolves to the node itself. (Ie, the root node's `root`
property refers to itself.)
* `node.isTop` True if this node is the top of its tree (ie, has no
`parent`, false otherwise).
* `node.top` The top node in this node's tree. This will be equal to
`node.root` for simple trees, but link targets will frequently be outside
of (or nested somewhere within) a `node_modules` hierarchy, and so will
have a different `top`.
* `node.dev`, `node.optional`, `node.devOptional`, `node.peer`, Indicators
as to whether this node is a dev, optional, and/or peer dependency.
These flags are relevant when pruning dependencies out of the tree or
deciding what to reify. See **Package Dependency Flags** below for
explanations.
* `node.edgesOut` Edges in the dependency graph indicating nodes that this
node depends on, which resolve its dependencies.
* `node.edgesIn` Edges in the dependency graph indicating nodes that depend
on this node.
* `extraneous` True if this package is not required by any other for any
reason. False for top of tree.
* `node.resolve(name)` Identify the node that will be returned when code
in this package runs `require(name)`
* `node.errors` Array of errors encountered while parsing package.json or
version specifiers.
### class Link
Link objects represent a symbolic link within the `node_modules` folder.
They have most of the same properties and methods as `Node` objects, with a
few differences.
* `link.target` A Node object representing the package that the link
references. If this is a Node already present within the tree, then it
will be the same object. If it's outside of the tree, then it will be
treated as the top of its own tree.
* `link.isLink` Always true.
* `link.children` This is always an empty map, since links don't have their
own children directly.
### class Edge
Edge objects represent a dependency relationship a package node to the
point in the tree where the dependency will be loaded. As nodes are moved
within the tree, Edges automatically update to point to the appropriate
location.
* `new Edge({ from, type, name, spec })` Creates a new edge with the
specified fields. After instantiation, none of the fields can be
changed directly.
* `edge.from` The node that has the dependency.
* `edge.type` The type of dependency. One of `'prod'`, `'dev'`, `'peer'`,
or `'optional'`.
* `edge.name` The name of the dependency. Ie, the key in the
relevant `package.json` dependencies object.
* `edge.spec` The specifier that is required. This can be a version,
range, tag name, git url, or tarball URL. Any specifier allowed by npm
is supported.
* `edge.to` Automatically set to the node in the tree that matches the
`name` field.
* `edge.valid` True if `edge.to` satisfies the specifier.
* `edge.error` A string indicating the type of error if there is a problem,
or `null` if it's valid. Values, in order of precedence:
* `DETACHED` Indicates that the edge has been detached from its
`edge.from` node, typically because a new edge was created when a
dependency specifier was modified.
* `MISSING` Indicates that the dependency is unmet. Note that this is
_not_ set for unmet dependencies of the `optional` type.
* `PEER LOCAL` Indicates that a `peerDependency` is found in the
node's local `node_modules` folder, and the node is not the top of
the tree. This violates the `peerDependency` contract, because it
means that the dependency is not a peer.
* `INVALID` Indicates that the dependency does not satisfy `edge.spec`.
* `edge.reload()` Re-resolve to find the appropriate value for `edge.to`.
Called automatically from the `Node` class when the tree is mutated.
### Package Dependency Flags
The dependency type of a node can be determined efficiently by looking at
the `dev`, `optional`, and `devOptional` flags on the node object. These
are updated by arborist when necessary whenever the tree is modified in
such a way that the dependency graph can change, and are relevant when
pruning nodes from the tree.
| extraneous | peer | dev | optional | devOptional | meaning | prune? |
|:----------:|:----:|:---:|:--------:|:----------------:|:-------------------------------------------------------------------------------------------------|:-------------------------------------------------------|
| | | | | | production dep | never |
| X | N/A | N/A | N/A | N/A | nothing depends on this, it is trash | always |
| | | X | | X<br>not in lock | devDependency, or only depended<br>on by devDependencies | if pruning dev |
| | | | X | X<br>not in lock | optionalDependency, or only depended<br>on by optionalDeps | if pruning optional |
| | | X | X | X<br>not in lock | Optional dependency of dep(s) in the<br>dev hierarchy | if pruning EITHER<br>dev OR optional |
| | | | | X<br>in lock | BOTH a non-optional dep within the<br>dev hierarchy, AND a dep within<br>the optional hierarchy | if pruning BOTH<br>dev AND optional |
| | X | | | | peer dependency, or only depended<br>on by peer dependencies | if pruning peers |
| | X | X | | X<br>not in lock | peer dependency of dev node hierarchy | if pruning peer OR<br>dev deps |
| | X | | X | X<br>not in lock | peer dependency of optional nodes, or<br>peerOptional dep | if pruning peer OR<br>optional deps |
| | X | X | X | X<br>not in lock | peer optional deps of the dev dep hierarchy | if pruning peer OR<br>optional OR dev |
| | X | | | X<br>in lock | BOTH a non-optional peer dep within the<br>dev hierarchy, AND a peer optional dep | if pruning peer deps OR:<br>BOTH optional AND dev deps |
* If none of these flags are set, then the node is required by the
dependency and/or peerDependency hierarchy. It should not be pruned.
* If _both_ `node.dev` and `node.optional` are set, then the node is an
optional dependency of one of the packages in the devDependency
hierarchy. It should be pruned if _either_ dev or optional deps are
being removed.
* If `node.dev` is set, but `node.optional` is not, then the node is
required in the devDependency hierarchy. It should be pruned if dev
dependencies are being removed.
* If `node.optional` is set, but `node.dev` is not, then the node is
required in the optionalDependency hierarchy. It should be pruned if
optional dependencies are being removed.
* If `node.devOptional` is set, then the node is a (non-optional)
dependency within the devDependency hierarchy, _and_ a dependency
within the `optionalDependency` hierarchy. It should be pruned if
_both_ dev and optional dependencies are being removed.
* If `node.peer` is set, then all the same semantics apply as above, except
that the dep is brought in by a peer dep at some point, rather than a
normal non-peer dependency.
Note: `devOptional` is only set in the shrinkwrap/package-lock file if
_neither_ `dev` nor `optional` are set, as it would be redundant.
## BIN
Arborist ships with a cli that can be used to run arborist specific commands outside of the context of the npm CLI. This script is currently not part of the public API and is subject to breaking changes outside of major version bumps.
To see the usage run:
```
npx @npmcli/arborist --help
```
@@ -0,0 +1,19 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
module.exports = (options, time) => new Arborist(options)
.loadActual(options)
.then(time)
.then(async ({ timing, result: tree }) => {
printTree(tree)
if (options.save) {
await tree.meta.save()
}
if (options.saveHidden) {
tree.meta.hiddenLockfile = true
tree.meta.filename = options.path + '/node_modules/.package-lock.json'
await tree.meta.save()
}
return `read ${tree.inventory.size} deps in ${timing.ms}`
})
@@ -0,0 +1,51 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
const log = require('./lib/logging.js')
const Vuln = require('../lib/vuln.js')
const printReport = report => {
for (const vuln of report.values()) {
log.info(printVuln(vuln))
}
if (report.topVulns.size) {
log.info('\n# top-level vulnerabilities')
for (const vuln of report.topVulns.values()) {
log.info(printVuln(vuln))
}
}
}
const printVuln = vuln => {
return {
__proto__: { constructor: Vuln },
name: vuln.name,
issues: [...vuln.advisories].map(a => printAdvisory(a)),
range: vuln.simpleRange,
nodes: [...vuln.nodes].map(node => `${node.name} ${node.location || '#ROOT'}`),
...(vuln.topNodes.size === 0 ? {} : {
topNodes: [...vuln.topNodes].map(node => `${node.location || '#ROOT'}`),
}),
}
}
const printAdvisory = a => `${a.title}${a.url ? ' ' + a.url : ''}`
module.exports = (options, time) => {
const arb = new Arborist(options)
return arb
.audit(options)
.then(time)
.then(async ({ timing, result: tree }) => {
if (options.fix) {
printTree(tree)
}
printReport(arb.auditReport)
if (tree.meta && options.save) {
await tree.meta.save()
}
return options.fix
? `resolved ${tree.inventory.size} deps in ${timing.seconds}`
: `done in ${timing.seconds}`
})
}
@@ -0,0 +1,38 @@
const Arborist = require('../')
const log = require('./lib/logging.js')
module.exports = (options, time) => {
const query = options._.shift()
const a = new Arborist(options)
return a
.loadVirtual()
.then(tree => {
// only load the actual tree if the virtual one doesn't have modern metadata
if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2)) {
log.error('old metadata, load actual')
throw 'load actual'
} else {
log.error('meta ok, return virtual tree')
return tree
}
})
.catch(() => a.loadActual())
.then(time)
.then(({ timing, result: tree }) => {
if (!query) {
for (const node of tree.inventory.values()) {
if (node.package.funding) {
log.info(node.name, node.location, node.package.funding)
}
}
} else {
for (const node of tree.inventory.query('name', query)) {
if (node.package.funding) {
log.info(node.name, node.location, node.package.funding)
}
}
}
return `read ${tree.inventory.size} deps in ${timing.ms}`
})
}
@@ -0,0 +1,14 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
module.exports = (options, time) => new Arborist(options)
.buildIdealTree(options)
.then(time)
.then(async ({ timing, result: tree }) => {
printTree(tree)
if (tree.meta && options.save) {
await tree.meta.save()
}
return `resolved ${tree.inventory.size} deps in ${timing.seconds}`
})
@@ -0,0 +1,112 @@
#!/usr/bin/env node
const fs = require('node:fs')
const path = require('node:path')
const { time } = require('proc-log')
const { bin, arb: options } = require('./lib/options')
const version = require('../package.json').version
const usage = (message = '') => `Arborist - the npm tree doctor
Version: ${version}
${message && '\n' + message + '\n'}
# USAGE
arborist <cmd> [path] [options...]
# COMMANDS
* reify: reify ideal tree to node_modules (install, update, rm, ...)
* prune: prune the ideal tree and reify (like npm prune)
* ideal: generate and print the ideal tree
* actual: read and print the actual tree in node_modules
* virtual: read and print the virtual tree in the local shrinkwrap file
* shrinkwrap: load a local shrinkwrap and print its data
* audit: perform a security audit on project dependencies
* funding: query funding information in the local package tree. A second
positional argument after the path name can limit to a package name.
* license: query license information in the local package tree. A second
positional argument after the path name can limit to a license type.
* help: print this text
* version: print the version
# OPTIONS
Most npm options are supported, but in camelCase rather than css-case. For
example, instead of '--dry-run', use '--dryRun'.
Additionally:
* --loglevel=warn|--quiet will suppress the printing of package trees
* --logfile <file|bool> will output logs to a file
* --timing will show timing information
* Instead of 'npm install <pkg>', use 'arborist reify --add=<pkg>'.
The '--add=<pkg>' option can be specified multiple times.
* Instead of 'npm rm <pkg>', use 'arborist reify --rm=<pkg>'.
The '--rm=<pkg>' option can be specified multiple times.
* Instead of 'npm update', use 'arborist reify --update-all'.
* 'npm audit fix' is 'arborist audit --fix'
`
const commands = {
version: () => console.log(version),
help: () => console.log(usage()),
exit: () => {
process.exitCode = 1
console.error(
usage(`Error: command '${bin.command}' does not exist.`)
)
},
}
const commandFiles = fs.readdirSync(__dirname).filter((f) => path.extname(f) === '.js' && f !== __filename)
for (const file of commandFiles) {
const command = require(`./${file}`)
const name = path.basename(file, '.js')
const totalTime = `bin:${name}:init`
const scriptTime = `bin:${name}:script`
commands[name] = () => {
const timers = require('./lib/timers')
const log = require('./lib/logging')
log.info(name, options)
const timeEnd = time.start(totalTime)
const scriptEnd = time.start(scriptTime)
return command(options, (result) => {
scriptEnd()
return {
result,
timing: {
seconds: `${timers.get(scriptTime) / 1e9}s`,
ms: `${timers.get(scriptTime) / 1e6}ms`,
},
}
})
.then((result) => {
log.info(result)
return result
})
.catch((err) => {
process.exitCode = 1
log.error(err)
return err
})
.then((r) => {
timeEnd()
if (bin.loglevel !== 'silent') {
console[process.exitCode ? 'error' : 'log'](r)
}
return r
})
}
}
if (commands[bin.command]) {
commands[bin.command]()
} else {
commands.exit()
}
@@ -0,0 +1,77 @@
const { log } = require('proc-log')
const fs = require('node:fs')
const { dirname } = require('node:path')
const os = require('node:os')
const { inspect, format } = require('node:util')
const { bin: options } = require('./options.js')
// add a meta method to proc-log for passing optional
// metadata through to log handlers
const META = Symbol('meta')
const parseArgs = (...args) => {
const { [META]: isMeta } = args[args.length - 1] || {}
return isMeta
? [args[args.length - 1], ...args.slice(0, args.length - 1)]
: [{}, ...args]
}
log.meta = (meta = {}) => ({ [META]: true, ...meta })
const levels = new Map([
'silly',
'verbose',
'info',
'http',
'notice',
'warn',
'error',
'silent',
].map((level, index) => [level, index]))
const addLogListener = (write, { eol = os.EOL, loglevel = 'silly', colors = false } = {}) => {
const levelIndex = levels.get(loglevel)
const magenta = m => colors ? `\x1B[35m${m}\x1B[39m` : m
const dim = m => colors ? `\x1B[2m${m}\x1B[22m` : m
const red = m => colors ? `\x1B[31m${m}\x1B[39m` : m
const formatter = (level, ...args) => {
const depth = level === 'error' && args[0] && args[0].code === 'ERESOLVE' ? Infinity : 10
if (level === 'info' && args[0] === 'timeEnd') {
args[1] = dim(args[1])
} else if (level === 'error' && args[0] === 'timeError') {
args[1] = red(args[1])
}
const messages = args.map(a => typeof a === 'string' ? a : inspect(a, { depth, colors }))
const pref = `${process.pid} ${magenta(level)} `
return pref + format(...messages).trim().split('\n').join(`${eol}${pref}`) + eol
}
process.on('log', (...args) => {
const [meta, level, ...logArgs] = parseArgs(...args)
if (levelIndex <= levels.get(level) || meta.force) {
write(formatter(level, ...logArgs))
}
})
}
if (options.loglevel !== 'silent') {
addLogListener((v) => process.stderr.write(v), {
eol: '\n',
colors: options.colors,
loglevel: options.loglevel,
})
}
if (options.logfile) {
log.silly('logfile', options.logfile)
fs.mkdirSync(dirname(options.logfile), { recursive: true })
const fd = fs.openSync(options.logfile, 'a')
addLogListener((str) => fs.writeSync(fd, str))
}
module.exports = log
@@ -0,0 +1,123 @@
const nopt = require('nopt')
const path = require('node:path')
const has = (o, k) => Object.prototype.hasOwnProperty.call(o, k)
const cleanPath = (val) => {
const k = Symbol('key')
const data = {}
nopt.typeDefs.path.validate(data, k, val)
return data[k]
}
const parse = (...noptArgs) => {
const binOnlyOpts = {
command: String,
loglevel: String,
colors: Boolean,
timing: ['always', Boolean],
logfile: String,
}
const arbOpts = {
add: Array,
rm: Array,
omit: Array,
update: Array,
workspaces: Array,
global: Boolean,
force: Boolean,
'global-style': Boolean,
'prefer-dedupe': Boolean,
'legacy-peer-deps': Boolean,
'update-all': Boolean,
before: Date,
path: path,
cache: path,
...binOnlyOpts,
}
const short = {
quiet: ['--loglevel', 'warn'],
logs: ['--logfile', 'true'],
w: '--workspaces',
g: '--global',
f: '--force',
}
const defaults = {
// key order is important for command and path
// since they shift positional args
// command is 1st, path is 2nd
command: (o) => o.argv.remain.shift(),
path: (o) => cleanPath(o.argv.remain.shift() || '.'),
colors: has(process.env, 'NO_COLOR') ? false : !!process.stderr.isTTY,
loglevel: 'silly',
timing: (o) => o.loglevel === 'silly',
cache: `${process.env.HOME}/.npm/_cacache`,
}
const derived = [
// making update either `all` or an array of names but not both
({ updateAll: all, update: names, ...o }) => {
if (all || names) {
o.update = all != null ? { all } : { names }
}
return o
},
({ logfile, ...o }) => {
// logfile is parsed as a string so if its true or set but empty
// then set the default logfile
if (logfile === 'true' || logfile === '') {
logfile = `arb-log-${new Date().toISOString().replace(/[.:]/g, '_')}.log`
}
// then parse it the same as nopt parses other paths
if (logfile) {
o.logfile = cleanPath(logfile)
}
return o
},
]
const transforms = [
// Camelcase all top level keys
(o) => {
const entries = Object.entries(o).map(([k, v]) => [
k.replace(/-./g, s => s[1].toUpperCase()),
v,
])
return Object.fromEntries(entries)
},
// Set defaults on unset keys
(o) => {
for (const [k, v] of Object.entries(defaults)) {
if (!has(o, k)) {
o[k] = typeof v === 'function' ? v(o) : v
}
}
return o
},
// Set/unset derived values
...derived.map((derive) => (o) => derive(o) || o),
// Separate bin and arborist options
({ argv: { remain: _ }, ...o }) => {
const bin = { _ }
for (const k of Object.keys(binOnlyOpts)) {
if (has(o, k)) {
bin[k] = o[k]
delete o[k]
}
}
return { bin, arb: o }
},
]
let options = nopt(arbOpts, short, ...noptArgs)
for (const t of transforms) {
options = t(options)
}
return options
}
module.exports = parse()
@@ -0,0 +1,4 @@
const { inspect } = require('node:util')
const log = require('./logging.js')
module.exports = tree => log.info(inspect(tree.toJSON(), { depth: Infinity }))
@@ -0,0 +1,33 @@
const { bin: options } = require('./options.js')
const log = require('./logging.js')
const timers = new Map()
const finished = new Map()
process.on('time', (level, name) => {
if (level === 'start') {
if (timers.has(name)) {
throw new Error('conflicting timer! ' + name)
}
timers.set(name, process.hrtime.bigint())
} else if (level === 'end') {
if (!timers.has(name)) {
throw new Error('timer not started! ' + name)
}
const elapsed = Number(process.hrtime.bigint() - timers.get(name))
timers.delete(name)
finished.set(name, elapsed)
if (options.timing) {
log.info('timeEnd', `${name} ${elapsed / 1e9}s`, log.meta({ force: options.timing === 'always' }))
}
}
})
process.on('exit', () => {
for (const name of timers.keys()) {
log.error('timeError', 'Dangling timer:', name)
process.exitCode = 1
}
})
module.exports = finished
@@ -0,0 +1,48 @@
const localeCompare = require('@isaacs/string-locale-compare')('en')
const Arborist = require('../')
const log = require('./lib/logging.js')
module.exports = (options, time) => {
const query = options._.shift()
const a = new Arborist(options)
return a
.loadVirtual()
.then(tree => {
// only load the actual tree if the virtual one doesn't have modern metadata
if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2)) {
throw 'load actual'
} else {
return tree
}
}).catch((er) => {
log.error('loading actual tree', er)
return a.loadActual()
})
.then(time)
.then(({ result: tree }) => {
const output = []
if (!query) {
const set = []
for (const license of tree.inventory.query('license')) {
set.push([tree.inventory.query('license', license).size, license])
}
for (const [count, license] of set.sort((a, b) =>
a[1] && b[1] ? b[0] - a[0] || localeCompare(a[1], b[1])
: a[1] ? -1
: b[1] ? 1
: 0)) {
output.push(`${count} ${license}`)
log.info(count, license)
}
} else {
for (const node of tree.inventory.query('license', query === 'undefined' ? undefined : query)) {
const msg = `${node.name} ${node.location} ${node.package.description || ''}`
output.push(msg)
log.info(msg)
}
}
return output.join('\n')
})
}
@@ -0,0 +1,48 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
const log = require('./lib/logging.js')
const printDiff = diff => {
const { depth } = require('treeverse')
depth({
tree: diff,
visit: d => {
if (d.location === '') {
return
}
switch (d.action) {
case 'REMOVE':
log.info('REMOVE', d.actual.location)
break
case 'ADD':
log.info('ADD', d.ideal.location, d.ideal.resolved)
break
case 'CHANGE':
log.info('CHANGE', d.actual.location, {
from: d.actual.resolved,
to: d.ideal.resolved,
})
break
}
},
getChildren: d => d.children,
})
}
module.exports = (options, time) => {
const arb = new Arborist(options)
return arb
.prune(options)
.then(time)
.then(async ({ timing, result: tree }) => {
printTree(tree)
if (options.dryRun) {
printDiff(arb.diff)
}
if (tree.meta && options.save) {
await tree.meta.save()
}
return `resolved ${tree.inventory.size} deps in ${timing.seconds}`
})
}
@@ -0,0 +1,48 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
const log = require('./lib/logging.js')
const printDiff = diff => {
const { depth } = require('treeverse')
depth({
tree: diff,
visit: d => {
if (d.location === '') {
return
}
switch (d.action) {
case 'REMOVE':
log.info('REMOVE', d.actual.location)
break
case 'ADD':
log.info('ADD', d.ideal.location, d.ideal.resolved)
break
case 'CHANGE':
log.info('CHANGE', d.actual.location, {
from: d.actual.resolved,
to: d.ideal.resolved,
})
break
}
},
getChildren: d => d.children,
})
}
module.exports = (options, time) => {
const arb = new Arborist(options)
return arb
.reify(options)
.then(time)
.then(async ({ timing, result: tree }) => {
printTree(tree)
if (options.dryRun) {
printDiff(arb.diff)
}
if (tree.meta && options.save) {
await tree.meta.save()
}
return `resolved ${tree.inventory.size} deps in ${timing.seconds}`
})
}
@@ -0,0 +1,7 @@
const Shrinkwrap = require('../lib/shrinkwrap.js')
module.exports = (options, time) => Shrinkwrap
.load(options)
.then((s) => s.commit())
.then(time)
.then(({ result: s }) => JSON.stringify(s, 0, 2))
@@ -0,0 +1,14 @@
const Arborist = require('../')
const printTree = require('./lib/print-tree.js')
module.exports = (options, time) => new Arborist(options)
.loadVirtual()
.then(time)
.then(async ({ timing, result: tree }) => {
printTree(tree)
if (options.save) {
await tree.meta.save()
}
return `read ${tree.inventory.size} deps in ${timing.ms}`
})
@@ -0,0 +1,143 @@
// add and remove dependency specs to/from pkg manifest
const { log } = require('proc-log')
const localeCompare = require('@isaacs/string-locale-compare')('en')
const add = ({ pkg, add, saveBundle, saveType }) => {
for (const { name, rawSpec } of add) {
let addSaveType = saveType
// if the user does not give us a type, we infer which type(s)
// to keep based on the same order of priority we do when
// building the tree as defined in the _loadDeps method of
// the node class.
if (!addSaveType) {
addSaveType = inferSaveType(pkg, name)
}
if (addSaveType === 'prod') {
// a production dependency can only exist as production (rpj ensures it
// doesn't coexist w/ optional)
deleteSubKey(pkg, 'devDependencies', name, 'dependencies')
deleteSubKey(pkg, 'peerDependencies', name, 'dependencies')
} else if (addSaveType === 'dev') {
// a dev dependency may co-exist as peer, or optional, but not production
deleteSubKey(pkg, 'dependencies', name, 'devDependencies')
} else if (addSaveType === 'optional') {
// an optional dependency may co-exist as dev (rpj ensures it doesn't
// coexist w/ prod)
deleteSubKey(pkg, 'peerDependencies', name, 'optionalDependencies')
} else { // peer or peerOptional is all that's left
// a peer dependency may coexist as dev
deleteSubKey(pkg, 'dependencies', name, 'peerDependencies')
deleteSubKey(pkg, 'optionalDependencies', name, 'peerDependencies')
}
const depType = saveTypeMap.get(addSaveType)
pkg[depType] = pkg[depType] || {}
if (rawSpec !== '*' || pkg[depType][name] === undefined) {
pkg[depType][name] = rawSpec
}
if (addSaveType === 'optional') {
// Affordance for previous npm versions that require this behaviour
pkg.dependencies = pkg.dependencies || {}
pkg.dependencies[name] = pkg.optionalDependencies[name]
}
if (addSaveType === 'peer' || addSaveType === 'peerOptional') {
const pdm = pkg.peerDependenciesMeta || {}
if (addSaveType === 'peer' && pdm[name] && pdm[name].optional) {
pdm[name].optional = false
} else if (addSaveType === 'peerOptional') {
pdm[name] = pdm[name] || {}
pdm[name].optional = true
pkg.peerDependenciesMeta = pdm
}
// peerDeps are often also a devDep, so that they can be tested when
// using package managers that don't auto-install peer deps
if (pkg.devDependencies && pkg.devDependencies[name] !== undefined) {
pkg.devDependencies[name] = pkg.peerDependencies[name]
}
}
if (saveBundle && addSaveType !== 'peer' && addSaveType !== 'peerOptional') {
// keep it sorted, keep it unique
const bd = new Set(pkg.bundleDependencies || [])
bd.add(name)
pkg.bundleDependencies = [...bd].sort(localeCompare)
}
}
return pkg
}
// Canonical source of both the map between saveType and where it correlates to
// in the package, and the names of all our dependencies attributes
const saveTypeMap = new Map([
['dev', 'devDependencies'],
['optional', 'optionalDependencies'],
['prod', 'dependencies'],
['peerOptional', 'peerDependencies'],
['peer', 'peerDependencies'],
])
// Finds where the package is already in the spec and infers saveType from that
const inferSaveType = (pkg, name) => {
for (const saveType of saveTypeMap.keys()) {
if (hasSubKey(pkg, saveTypeMap.get(saveType), name)) {
if (
saveType === 'peerOptional' &&
(!hasSubKey(pkg, 'peerDependenciesMeta', name) ||
!pkg.peerDependenciesMeta[name].optional)
) {
return 'peer'
}
return saveType
}
}
return 'prod'
}
const hasSubKey = (pkg, depType, name) => {
return pkg[depType] && Object.prototype.hasOwnProperty.call(pkg[depType], name)
}
// Removes a subkey and warns about it if it's being replaced
const deleteSubKey = (pkg, depType, name, replacedBy) => {
if (hasSubKey(pkg, depType, name)) {
if (replacedBy) {
log.warn('idealTree', `Removing ${depType}.${name} in favor of ${replacedBy}.${name}`)
}
delete pkg[depType][name]
// clean up peerDepsMeta if we are removing something from peerDependencies
if (depType === 'peerDependencies' && pkg.peerDependenciesMeta) {
delete pkg.peerDependenciesMeta[name]
if (!Object.keys(pkg.peerDependenciesMeta).length) {
delete pkg.peerDependenciesMeta
}
}
if (!Object.keys(pkg[depType]).length) {
delete pkg[depType]
}
}
}
const rm = (pkg, rm) => {
for (const depType of new Set(saveTypeMap.values())) {
for (const name of rm) {
deleteSubKey(pkg, depType, name)
}
}
if (pkg.bundleDependencies) {
pkg.bundleDependencies = pkg.bundleDependencies
.filter(name => !rm.includes(name))
if (!pkg.bundleDependencies.length) {
delete pkg.bundleDependencies
}
}
return pkg
}
module.exports = { add, rm, saveTypeMap, hasSubKey }
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,322 @@
// The arborist manages three trees:
// - actual
// - virtual
// - ideal
//
// The actual tree is what's present on disk in the node_modules tree
// and elsewhere that links may extend.
//
// The virtual tree is loaded from metadata (package.json and lock files).
//
// The ideal tree is what we WANT that actual tree to become. This starts
// with the virtual tree, and then applies the options requesting
// add/remove/update actions.
//
// To reify a tree, we calculate a diff between the ideal and actual trees,
// and then turn the actual tree into the ideal tree by taking the actions
// required. At the end of the reification process, the actualTree is
// updated to reflect the changes.
//
// Each tree has an Inventory at the root. Shrinkwrap is tracked by Arborist
// instance. It always refers to the actual tree, but is updated (and written
// to disk) on reification.
// Each of the mixin "classes" adds functionality, but are not dependent on
// constructor call order. So, we just load them in an array, and build up
// the base class, so that the overall voltron class is easier to test and
// cover, and separation of concerns can be maintained.
const { resolve } = require('node:path')
const { homedir } = require('node:os')
const { depth } = require('treeverse')
const mapWorkspaces = require('@npmcli/map-workspaces')
const { log, time } = require('proc-log')
const { saveTypeMap } = require('../add-rm-pkg-deps.js')
const AuditReport = require('../audit-report.js')
const relpath = require('../relpath.js')
const PackumentCache = require('../packument-cache.js')
const mixins = [
require('../tracker.js'),
require('./build-ideal-tree.js'),
require('./load-actual.js'),
require('./load-virtual.js'),
require('./rebuild.js'),
require('./reify.js'),
require('./isolated-reifier.js'),
]
const _setWorkspaces = Symbol.for('setWorkspaces')
const Base = mixins.reduce((a, b) => b(a), require('node:events'))
// if it's 1, 2, or 3, set it explicitly that.
// if undefined or null, set it null
// otherwise, throw.
const lockfileVersion = lfv => {
if (lfv === 1 || lfv === 2 || lfv === 3) {
return lfv
}
if (lfv === undefined || lfv === null) {
return null
}
throw new TypeError('Invalid lockfileVersion config: ' + lfv)
}
class Arborist extends Base {
constructor (options = {}) {
const timeEnd = time.start('arborist:ctor')
super(options)
// normalize trailing slash
const registry = options.registry || 'https://registry.npmjs.org'
options.registry = this.registry = registry.replace(/(?<!\/)\/+$/, '') + '/'
// TODO as we consolidate constructors it's more apparent that we are not parsing options and using this.options consistently
const {
actualTree,
global,
idealTree = null,
installLinks = false,
legacyPeerDeps = false,
virtualTree,
workspaces,
} = options
if (workspaces?.length && global) {
throw new Error('Cannot operate on workspaces in global mode')
}
// the tree of nodes on disk
this.actualTree = actualTree
this.idealTree = idealTree
this.installLinks = installLinks
this.legacyPeerDeps = legacyPeerDeps
// the virtual tree we load from a shrinkwrap
this.virtualTree = virtualTree
this.options = {
nodeVersion: process.version,
...options,
Arborist: this.constructor,
allowScripts: options.allowScripts ?? null,
binLinks: 'binLinks' in options ? !!options.binLinks : true,
cache: options.cache || `${homedir()}/.npm/_cacache`,
dangerouslyAllowAllScripts: !!options.dangerouslyAllowAllScripts,
dryRun: !!options.dryRun,
formatPackageLock: 'formatPackageLock' in options ? !!options.formatPackageLock : true,
force: !!options.force,
global: !!options.global,
ignoreScripts: !!options.ignoreScripts,
installStrategy: options.global ? 'shallow' : (options.installStrategy ? options.installStrategy : 'hoisted'),
lockfileVersion: lockfileVersion(options.lockfileVersion),
packageLockOnly: !!options.packageLockOnly,
packumentCache: options.packumentCache || new PackumentCache(),
path: options.path || '.',
rebuildBundle: 'rebuildBundle' in options ? !!options.rebuildBundle : true,
replaceRegistryHost: options.replaceRegistryHost,
savePrefix: 'savePrefix' in options ? options.savePrefix : '^',
scriptShell: options.scriptShell,
usePackageLock: 'packageLock' in options ? options.packageLock : true,
workspaces: options.workspaces || [],
workspacesEnabled: options.workspacesEnabled !== false,
}
// TODO we only ever look at this.options.replaceRegistryHost, not
// this.replaceRegistryHost. Defaulting needs to be written back to
// this.options to work properly
this.replaceRegistryHost = this.options.replaceRegistryHost =
(!this.options.replaceRegistryHost || this.options.replaceRegistryHost === 'npmjs') ?
'registry.npmjs.org' : this.options.replaceRegistryHost
if (options.saveType && !saveTypeMap.get(options.saveType)) {
throw new Error(`Invalid saveType ${options.saveType}`)
}
this.cache = resolve(this.options.cache)
this.diff = null
this.path = resolve(this.options.path)
this.scriptsRun = new Set()
timeEnd()
}
// TODO: We should change these to static functions instead
// of methods for the next major version
// Get the actual nodes corresponding to a root node's child workspaces,
// given a list of workspace names.
workspaceNodes (tree, workspaces) {
const wsMap = tree.workspaces
if (!wsMap) {
log.warn('workspaces', 'filter set, but no workspaces present')
return []
}
const nodes = []
for (const name of workspaces) {
const path = wsMap.get(name)
if (!path) {
log.warn('workspaces', `${name} in filter set, but not in workspaces`)
continue
}
const loc = relpath(tree.realpath, path)
const node = tree.inventory.get(loc)
if (!node) {
log.warn('workspaces', `${name} in filter set, but no workspace folder present`)
continue
}
nodes.push(node)
}
return nodes
}
// returns a set of workspace nodes and all their deps
// TODO why is includeWorkspaceRoot a param?
// TODO why is workspaces a param?
workspaceDependencySet (tree, workspaces, includeWorkspaceRoot) {
const wsNodes = this.workspaceNodes(tree, workspaces)
if (includeWorkspaceRoot) {
for (const edge of tree.edgesOut.values()) {
if (edge.type !== 'workspace' && edge.to) {
wsNodes.push(edge.to)
}
}
}
const wsDepSet = new Set(wsNodes)
const extraneous = new Set()
for (const node of wsDepSet) {
for (const edge of node.edgesOut.values()) {
const dep = edge.to
if (dep) {
wsDepSet.add(dep)
if (dep.isLink) {
wsDepSet.add(dep.target)
}
}
}
for (const child of node.children.values()) {
if (child.extraneous) {
extraneous.add(child)
}
}
}
for (const extra of extraneous) {
wsDepSet.add(extra)
}
return wsDepSet
}
// returns a set of root dependencies, excluding dependencies that are
// exclusively workspace dependencies
excludeWorkspacesDependencySet (tree) {
const rootDepSet = new Set()
depth({
tree,
visit: node => {
for (const { to } of node.edgesOut.values()) {
if (!to || to.isWorkspace) {
continue
}
for (const edgeIn of to.edgesIn.values()) {
if (edgeIn.from.isRoot || rootDepSet.has(edgeIn.from)) {
rootDepSet.add(to)
}
}
}
return node
},
filter: node => node,
getChildren: (node, tree) =>
[...tree.edgesOut.values()].map(edge => edge.to),
})
return rootDepSet
}
async [_setWorkspaces] (node) {
const workspaces = await mapWorkspaces({
cwd: node.path,
pkg: node.package,
})
if (node && workspaces.size) {
node.workspaces = workspaces
}
return node
}
async audit (options = {}) {
this.addTracker('audit')
if (this.options.global) {
throw Object.assign(
new Error('`npm audit` does not support testing globals'),
{ code: 'EAUDITGLOBAL' }
)
}
// allow the user to set options on the ctor as well.
// XXX: deprecate separate method options objects.
options = { ...this.options, ...options }
const timeEnd = time.start('audit')
let tree
if (options.packageLock === false) {
// build ideal tree
await this.loadActual(options)
await this.buildIdealTree()
tree = this.idealTree
} else {
tree = await this.loadVirtual()
}
if (this.options.workspaces.length) {
options.filterSet = this.workspaceDependencySet(
tree,
this.options.workspaces,
this.options.includeWorkspaceRoot
)
}
if (!options.workspacesEnabled) {
options.filterSet =
this.excludeWorkspacesDependencySet(tree)
}
this.auditReport = await AuditReport.load(tree, options)
const ret = options.fix ? this.reify(options) : this.auditReport
timeEnd()
this.finishTracker('audit')
return ret
}
// Build an ideal tree (or reuse an already-built one) and return the
// resulting lockfile contents as a string, without writing to disk.
// Useful for callers that want to inspect, diff, or store a lockfile
// somewhere other than the project's `package-lock.json`.
async lockfileString (options = {}) {
await this.buildIdealTree(options)
return this.idealTree.meta.toString(options)
}
async dedupe (options = {}) {
// allow the user to set options on the ctor as well.
// XXX: deprecate separate method options objects.
options = { ...this.options, ...options }
const tree = await this.loadVirtual().catch(() => this.loadActual())
const names = []
for (const name of tree.inventory.query('name')) {
if (tree.inventory.query('name', name).size > 1) {
names.push(name)
}
}
return this.reify({
...options,
preferDedupe: true,
update: { names },
})
}
}
module.exports = Arborist
@@ -0,0 +1,480 @@
const { mkdirSync } = require('node:fs')
const pacote = require('pacote')
const { join } = require('node:path')
const { depth } = require('treeverse')
const crypto = require('node:crypto')
const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js')
// generate short hash key based on the dependency tree starting at this node
const getKey = (startNode) => {
const deps = []
const branch = []
depth({
tree: startNode,
getChildren: node => node.dependencies,
visit: node => {
branch.push(`${node.packageName}@${node.version}`)
deps.push(`${branch.join('->')}::${node.resolved}`)
},
leave: () => {
branch.pop()
},
})
deps.sort()
// TODO these replaces were originally to deal with node 14 not supporting base64url and likely don't need to happen anymore
// Changing this is a pretty significant breaking change, but removing parts of the hash increases collision possibilities (even if slight).
const hash = crypto.createHash('shake256', { outputLength: 16 })
.update(deps.join(','))
.digest('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/m, '')
return `${startNode.packageName}@${startNode.version}-${hash}`
}
module.exports = cls => class IsolatedReifier extends cls {
#externalProxies = new Map()
#omit = new Set()
#rootDeclaredDeps = new Set()
#processedEdges = new Set()
#workspaceProxies = new Map()
#generateChild (node, location, pkg, isInStore, root) {
const newChild = new IsolatedNode({
isInStore,
location,
name: node.packageName || node.name,
optional: node.optional,
package: pkg,
parent: root,
path: join(this.idealGraph.localPath, location),
resolved: node.resolved,
root,
})
// XXX top is from place-dep not lib/node.js
newChild.top = { path: this.idealGraph.localPath }
root.children.set(newChild.location, newChild)
root.inventory.set(newChild.location, newChild)
}
/**
* Create an ideal graph.
*
* An implementation of npm RFC-0042
* https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md
*
* This entire file should be considered technical debt that will be resolved with an Arborist refactor or rewrite.
* Embedded logic in Nodes and Links, and the incremental state of building trees and reifying contains too many assumptions to do a linked mode properly.
*
* Instead, this approach takes a tree built from build-ideal-tree, and returns a new tree-like structure without the embedded logic of Node and Link classes.
*
* Since the RFC requires leaving the package-lock in place, this approach temporarily replaces the tree state for a couple of steps of reifying.
*
**/
async makeIdealGraph () {
const idealTree = this.idealTree
this.#omit = new Set(this.options.omit)
const omit = this.#omit
// npm auto-creates 'workspace' edges from root to all workspaces.
// For isolated/linked mode, only include workspaces that root explicitly declares as dependencies.
// When omitting dep types, exclude those from the declared set so their workspaces aren't hoisted.
const rootPkg = idealTree.package
this.#rootDeclaredDeps = new Set(Object.keys(Object.assign({},
rootPkg.dependencies,
(!omit.has('dev') && rootPkg.devDependencies),
(!omit.has('optional') && rootPkg.optionalDependencies),
(!omit.has('peer') && rootPkg.peerDependencies)
)))
// XXX this sometimes acts like a node too
this.idealGraph = {
external: [],
isProjectRoot: true,
localLocation: idealTree.location,
localPath: idealTree.path,
path: idealTree.path,
}
this.counter = 0
// Skip extraneous fsChildren: workspaces removed from the root manifest can linger in fsChildren via the lockfile, and re-materializing them here would re-create a directory the user just deleted.
const fsChildren = Array.from(idealTree.fsChildren.values()).filter(w => !w.extraneous)
this.idealGraph.workspaces = await Promise.all(fsChildren.map(w => this.#workspaceProxy(w)))
const processed = new Set()
const queue = [idealTree, ...idealTree.fsChildren]
while (queue.length !== 0) {
const next = queue.pop()
if (processed.has(next.location)) {
continue
}
processed.add(next.location)
next.edgesOut.forEach(edge => {
if (edge.to && !(next.package.bundleDependencies || next.package.bundledDependencies || []).includes(edge.to.name) && !edge.to.shouldOmit?.(omit)) {
queue.push(edge.to)
}
})
// local `file:` deps are in fsChildren but are not workspaces.
// they are already handled as workspace-like proxies above and should not go through the external/store extraction path.
// Links with file: resolved paths (from `npm link`) should also be treated as local dependencies and symlinked directly instead of being extracted into the store.
const isLocalFileDep = next.isLink && next.resolved?.startsWith('file:')
if (isLocalFileDep && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
this.idealGraph.workspaces.push(await this.#workspaceProxy(next.target))
} else if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
this.idealGraph.external.push(await this.#externalProxy(next))
}
}
await this.#assignCommonProperties(idealTree, this.idealGraph)
}
async #workspaceProxy (node) {
if (this.#workspaceProxies.has(node)) {
return this.#workspaceProxies.get(node)
}
const result = {}
// XXX this goes recursive if we don't set here because assignCommonProperties also calls this.#workspaceProxy
this.#workspaceProxies.set(node, result)
result.localLocation = node.location
result.localPath = node.path
result.isWorkspace = true
result.resolved = node.resolved
await this.#assignCommonProperties(node, result)
return result
}
async #externalProxy (node) {
if (this.#externalProxies.has(node)) {
return this.#externalProxies.get(node)
}
const result = {}
// XXX this goes recursive if we don't set here because assignCommonProperties also calls this.#externalProxy
this.#externalProxies.set(node, result)
await this.#assignCommonProperties(node, result, !node.hasShrinkwrap)
if (node.hasShrinkwrap) {
const dir = join(
node.root.path,
'node_modules',
'.store',
`${node.packageName}@${node.version}`
)
mkdirSync(dir, { recursive: true })
// TODO this approach feels wrong and shouldn't be necessary for shrinkwraps
await pacote.extract(node.resolved, dir, {
...this.options,
resolved: node.resolved,
integrity: node.integrity,
// TODO _isRoot
})
const Arborist = this.constructor
const arb = new Arborist({ ...this.options, path: dir })
// Make sure that the ideal tree is build as the rest of the algorithm depends on it.
await arb.buildIdealTree({
complete: false,
dev: false,
})
await arb.makeIdealGraph()
this.idealGraph.external.push(...arb.idealGraph.external)
for (const edge of arb.idealGraph.external) {
edge.root = this.idealGraph
edge.id = `${node.id}=>${edge.id}`
}
result.localDependencies = []
result.externalDependencies = arb.idealGraph.externalDependencies
result.externalOptionalDependencies = arb.idealGraph.externalOptionalDependencies
result.dependencies = [
...result.externalDependencies,
...result.externalOptionalDependencies,
]
}
result.optional = node.optional
result.resolved = node.resolved
result.version = node.version
return result
}
async #assignCommonProperties (node, result, populateDeps = true) {
result.root = this.idealGraph
// XXX does anything need this?
result.id = this.counter++
/* istanbul ignore next - packageName is always set for real packages */
result.name = result.isWorkspace ? (node.packageName || node.name) : node.name
result.packageName = node.packageName || node.name
result.package = { ...node.package }
result.package.bundleDependencies = undefined
if (!populateDeps) {
return
}
let edges = [...node.edgesOut.values()].filter(edge =>
edge.to?.target &&
!(node.package.bundledDependencies || node.package.bundleDependencies)?.includes(edge.to.name)
)
// Only omit edge types for root and workspace nodes (matching shouldOmit scope)
if ((node.isProjectRoot || node.isWorkspace) && this.#omit.size) {
edges = edges.filter(edge => {
if (edge.dev && this.#omit.has('dev')) {
return false
}
if (edge.optional && this.#omit.has('optional')) {
return false
}
if (edge.peer && this.#omit.has('peer')) {
return false
}
return true
})
}
let nonOptionalDeps = edges.filter(edge => !edge.optional).map(edge => edge.to.target)
// npm auto-creates 'workspace' edges from root to all workspaces.
// For isolated/linked mode, only include workspaces that root explicitly declares as dependencies.
if (node.isProjectRoot) {
nonOptionalDeps = nonOptionalDeps.filter(n => !n.isWorkspace || this.#rootDeclaredDeps.has(n.packageName))
}
// When legacyPeerDeps is enabled, peer dep edges are not created on the node.
// Resolve them from the tree so they get symlinked in the store.
const peerDeps = node.package.peerDependencies
if (peerDeps && node.legacyPeerDeps) {
const edgeNames = new Set(edges.map(edge => edge.name))
for (const peerName in peerDeps) {
if (!edgeNames.has(peerName)) {
const resolved = node.resolve(peerName)
if (resolved && resolved !== node && !resolved.inert) {
nonOptionalDeps.push(resolved.target)
}
}
}
}
// local `file:` deps (non-workspace fsChildren) should be treated as local dependencies, not external, so they get symlinked directly instead of being extracted into the store.
const isLocal = (n) => n.isWorkspace || node.fsChildren?.has(n)
const optionalDeps = edges.filter(edge => edge.optional).map(edge => edge.to.target)
result.localDependencies = await Promise.all(nonOptionalDeps.filter(isLocal).map(n => this.#workspaceProxy(n)))
result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !isLocal(n) && !n.inert).map(n => this.#externalProxy(n)))
result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.inert).map(n => this.#externalProxy(n)))
result.dependencies = [
...result.externalDependencies,
...result.localDependencies,
...result.externalOptionalDependencies,
]
}
async #createBundledTree () {
// TODO: make sure that idealTree object exists
const idealTree = this.idealTree
// TODO: test workspaces having bundled deps
const queue = []
for (const [, edge] of idealTree.edgesOut) {
if (edge.to && (idealTree.package.bundleDependencies || idealTree.package.bundledDependencies || []).includes(edge.to.name)) {
queue.push({ from: idealTree, to: edge.to })
}
}
for (const child of idealTree.fsChildren) {
for (const [, edge] of child.edgesOut) {
if (edge.to && (child.package.bundleDependencies || child.package.bundledDependencies || []).includes(edge.to.name)) {
queue.push({ from: child, to: edge.to })
}
}
}
const processed = new Set()
const nodes = new Map()
const edges = []
while (queue.length !== 0) {
const nextEdge = queue.pop()
const key = `${nextEdge.from.location}=>${nextEdge.to.location}`
// should be impossible, unless bundled is duped
/* istanbul ignore next */
if (processed.has(key)) {
continue
}
processed.add(key)
const from = nextEdge.from
if (!from.isRoot && !from.isWorkspace) {
nodes.set(from.location, { location: from.location, resolved: from.resolved, name: from.name, optional: from.optional, pkg: { ...from.package, bundleDependencies: undefined } })
}
const to = nextEdge.to
nodes.set(to.location, { location: to.location, resolved: to.resolved, name: to.name, optional: to.optional, pkg: { ...to.package, bundleDependencies: undefined } })
edges.push({ from: from.isRoot ? 'root' : from.location, to: to.location })
to.edgesOut.forEach(edge => {
// an edge out should always have a to
/* istanbul ignore else */
if (edge.to) {
queue.push({ from: edge.from, to: edge.to })
}
})
}
return { edges, nodes }
}
async createIsolatedTree () {
await this.makeIdealGraph()
const bundledTree = await this.#createBundledTree()
const root = new IsolatedNode(this.idealGraph)
root.root = root
root.inventory.set('', root)
const processed = new Set()
for (const c of this.idealGraph.workspaces) {
const wsName = c.packageName
// XXX parent? root?
const workspace = new IsolatedNode({
location: c.localLocation,
name: wsName,
package: c.package,
path: c.localPath,
resolved: c.resolved,
})
root.fsChildren.add(workspace)
root.inventory.set(workspace.location, workspace)
root.workspaces.set(wsName, workspace.path)
// Declared workspaces are symlinked at root node_modules/.
// Undeclared workspaces get a tree-only Link kept for diff/filter participation but not materialized on disk.
const isDeclared = this.#rootDeclaredDeps.has(wsName)
const wsLink = new IsolatedLink({
location: isDeclared ? join('node_modules', wsName) : join(c.localLocation, 'node_modules', wsName),
name: wsName,
package: workspace.package,
parent: root,
path: isDeclared ? join(root.path, 'node_modules', wsName) : join(root.path, c.localLocation, 'node_modules', wsName),
realpath: workspace.path,
root,
target: workspace,
})
if (!isDeclared) {
wsLink.isUndeclaredWorkspaceLink = true
}
root.children.set(wsName, wsLink)
root.inventory.set(wsLink.location, wsLink)
workspace.linksIn.add(wsLink)
}
this.idealGraph.external.forEach(c => {
const key = getKey(c)
if (processed.has(key)) {
return
}
processed.add(key)
const location = join('node_modules', '.store', key, 'node_modules', c.packageName)
this.#generateChild(c, location, c.package, true, root)
})
bundledTree.nodes.forEach(node => {
this.#generateChild(node, node.location, node.pkg, false, root)
})
bundledTree.edges.forEach(edge => {
const from = edge.from === 'root' ? root : root.inventory.get(edge.from)
const to = root.inventory.get(edge.to)
// Maybe optional should be propagated from the original edge
const newEdge = { optional: false, from, to }
from.edgesOut.set(to.name, newEdge)
to.edgesIn.add(newEdge)
})
this.#processEdges(this.idealGraph, false, root)
for (const node of this.idealGraph.workspaces) {
this.#processEdges(node, false, root)
}
return root
}
#processEdges (node, externalEdge, root) {
const key = getKey(node)
if (this.#processedEdges.has(key)) {
return
}
this.#processedEdges.add(key)
let from, nmFolder
if (externalEdge) {
const fromLocation = join('node_modules', '.store', key, 'node_modules', node.packageName)
from = root.children.get(fromLocation)
nmFolder = join('node_modules', '.store', key, 'node_modules')
} else {
from = node.isProjectRoot ? root : root.inventory.get(node.localLocation)
nmFolder = join(node.localLocation, 'node_modules')
}
/* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */
if (!from) {
return
}
for (const dep of node.localDependencies) {
this.#processEdges(dep, false, root)
// nonOptional, local
this.#processDeps(dep, false, false, root, from, nmFolder)
}
for (const dep of node.externalDependencies) {
this.#processEdges(dep, true, root)
// nonOptional, external
this.#processDeps(dep, false, true, root, from, nmFolder)
}
for (const dep of node.externalOptionalDependencies) {
this.#processEdges(dep, true, root)
// optional, external
this.#processDeps(dep, true, true, root, from, nmFolder)
}
}
#processDeps (dep, optional, external, root, from, nmFolder) {
const toKey = getKey(dep)
let target
if (external) {
const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.packageName)
target = root.children.get(toLocation)
} else {
target = root.inventory.get(dep.localLocation)
}
// TODO: we should no-op is an edge has already been created with the same fromKey and toKey
/* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */
if (!target) {
return
}
if (dep.package.bin) {
for (const bn in dep.package.bin) {
target.binPaths.push(join(dep.root.localPath, nmFolder, '.bin', bn))
}
}
const pkg = {
_id: dep.package._id,
bin: target.package.bin,
bundleDependencies: undefined,
deprecated: undefined,
scripts: dep.package.scripts,
version: dep.package.version,
}
const link = new IsolatedLink({
isStoreLink: true,
location: join(nmFolder, dep.name),
name: toKey,
optional,
parent: root,
package: pkg,
path: join(dep.root.localPath, nmFolder, dep.name),
realpath: target.path,
resolved: external ? `file:.store/${toKey}/node_modules/${dep.packageName}` : dep.resolved,
root,
target,
})
// XXX top is from place-dep not lib/link.js
link.top = { path: dep.root.localPath }
const newEdge1 = { optional, from, to: link }
from.edgesOut.set(dep.name, newEdge1)
link.edgesIn.add(newEdge1)
const newEdge2 = { optional: false, from: link, to: target }
link.edgesOut.set(dep.name, newEdge2)
target.edgesIn.add(newEdge2)
root.children.set(link.location, link)
}
}
@@ -0,0 +1,429 @@
// mix-in implementing the loadActual method
const { dirname, join, normalize, relative, resolve } = require('node:path')
const PackageJson = require('@npmcli/package-json')
const { readdirScoped } = require('@npmcli/fs')
const { walkUp } = require('walk-up-path')
const { commonAncestorPath } = require('common-ancestor-path')
const treeCheck = require('../tree-check.js')
const Shrinkwrap = require('../shrinkwrap.js')
const calcDepFlags = require('../calc-dep-flags.js')
const Node = require('../node.js')
const Link = require('../link.js')
const realpath = require('../realpath.js')
// public symbols
const _changePath = Symbol.for('_changePath')
const _setWorkspaces = Symbol.for('setWorkspaces')
const _rpcache = Symbol.for('realpathCache')
const _stcache = Symbol.for('statCache')
module.exports = cls => class ActualLoader extends cls {
#actualTree
// ensure when walking the tree that we don't call loadTree on the same
// actual node more than one time.
#actualTreeLoaded = new Set()
#actualTreePromise
// cache of nodes when loading the actualTree, so that we avoid loaded the
// same node multiple times when symlinks attack.
#cache = new Map()
#filter
// cache of link targets for setting fsParent links
// We don't do fsParent as a magic getter/setter, because it'd be too costly
// to keep up to date along the walk.
// And, we know that it can ONLY be relevant when the node is a target of a
// link; otherwise, it'd be in a node_modules folder, so take advantage of
// that to limit the scans later.
#topNodes = new Set()
#transplantFilter
// public method
// TODO remove options param in next semver major
async loadActual (options = {}) {
// In the past this.actualTree was set as a promise that eventually
// resolved, and overwrite this.actualTree with the resolved value. This
// was a problem because virtually no other code expects this.actualTree to
// be a promise. Instead we only set it once resolved, and also return it
// from the promise so that it is what's returned from this function when
// awaited.
if (this.actualTree) {
return this.actualTree
}
if (!this.#actualTreePromise) {
// allow the user to set options on the ctor as well.
// XXX: deprecate separate method options objects.
options = { ...this.options, ...options }
this.#actualTreePromise = this.#loadActual(options)
.then(tree => {
// reset all deps to extraneous prior to recalc
if (!options.root) {
for (const node of tree.inventory.values()) {
node.extraneous = true
}
}
// only reset root flags if we're not re-rooting,
// otherwise leave as-is
calcDepFlags(tree, !options.root)
this.actualTree = treeCheck(tree)
return this.actualTree
})
}
return this.#actualTreePromise
}
// return the promise so that we don't ever have more than one going at the
// same time. This is so that buildIdealTree can default to the actualTree
// if no shrinkwrap present, but reify() can still call buildIdealTree and
// loadActual in parallel safely.
async #loadActual (options) {
// mostly realpath to throw if the root doesn't exist
const {
global,
filter = () => true,
root = null,
transplantFilter = () => true,
ignoreMissing = false,
forceActual = false,
} = options
this.#filter = filter
this.#transplantFilter = transplantFilter
if (global) {
const real = await realpath(this.path, this[_rpcache], this[_stcache])
const params = {
path: this.path,
realpath: real,
pkg: {},
global,
loadOverrides: true,
}
if (this.path === real) {
this.#actualTree = this.#newNode(params)
} else {
this.#actualTree = await this.#newLink(params)
}
} else {
// not in global mode, hidden lockfile is allowed, load root pkg too
this.#actualTree = await this.#loadFSNode({
path: this.path,
real: await realpath(this.path, this[_rpcache], this[_stcache]),
loadOverrides: true,
})
this.#actualTree.assertRootOverrides()
// if forceActual is set, don't even try the hidden lockfile
if (!forceActual) {
// Note: hidden lockfile will be rejected if it's not the latest thing
// in the folder, or if any of the entries in the hidden lockfile are
// missing.
const meta = await Shrinkwrap.load({
path: this.#actualTree.path,
hiddenLockfile: true,
resolveOptions: this.options,
})
if (meta.loadedFromDisk) {
this.#actualTree.meta = meta
// have to load on a new Arborist object, so we don't assign
// the virtualTree on this one! Also, the weird reference is because
// we can't easily get a ref to Arborist in this module, without
// creating a circular reference, since this class is a mixin used
// to build up the Arborist class itself.
await new this.constructor({ ...this.options }).loadVirtual({
root: this.#actualTree,
})
await this[_setWorkspaces](this.#actualTree)
this.#transplant(root)
return this.#actualTree
}
}
const meta = await Shrinkwrap.load({
path: this.#actualTree.path,
lockfileVersion: this.options.lockfileVersion,
resolveOptions: this.options,
})
this.#actualTree.meta = meta
}
await this.#loadFSTree(this.#actualTree)
await this[_setWorkspaces](this.#actualTree)
// if there are workspace targets without Link nodes created, load
// the targets, so that we know what they are.
if (this.#actualTree.workspaces && this.#actualTree.workspaces.size) {
const promises = []
for (const path of this.#actualTree.workspaces.values()) {
if (!this.#cache.has(path)) {
// workspace overrides use the root overrides
const p = this.#loadFSNode({ path, root: this.#actualTree, useRootOverrides: true })
.then(node => this.#loadFSTree(node))
promises.push(p)
}
}
await Promise.all(promises)
}
if (!ignoreMissing) {
await this.#findMissingEdges()
}
// try to find a node that is the parent in a fs tree sense, but not a
// node_modules tree sense, of any link targets. this allows us to
// resolve deps that node will find, but a legacy npm view of the
// world would not have noticed.
for (const path of this.#topNodes) {
const node = this.#cache.get(path)
if (node && !node.parent && !node.fsParent) {
for (const p of walkUp(dirname(path))) {
if (this.#cache.has(p)) {
node.fsParent = this.#cache.get(p)
break
}
}
}
}
this.#transplant(root)
if (global) {
// need to depend on the children, or else all of them
// will end up being flagged as extraneous, since the
// global root isn't a "real" project
const tree = this.#actualTree
const actualRoot = tree.isLink ? tree.target : tree
const { dependencies = {} } = actualRoot.package
for (const [name, kid] of actualRoot.children.entries()) {
const def = kid.isLink ? `file:${kid.realpath}` : '*'
dependencies[name] = dependencies[name] || def
}
actualRoot.package = { ...actualRoot.package, dependencies }
}
return this.#actualTree
}
#transplant (root) {
if (!root || root === this.#actualTree) {
return
}
this.#actualTree[_changePath](root.path)
for (const node of this.#actualTree.children.values()) {
if (!this.#transplantFilter(node)) {
node.root = null
}
}
root.replace(this.#actualTree)
for (const node of this.#actualTree.fsChildren) {
node.root = this.#transplantFilter(node) ? root : null
}
this.#actualTree = root
}
async #loadFSNode ({ path, parent, real, root, loadOverrides, useRootOverrides }) {
if (!real) {
try {
real = await realpath(path, this[_rpcache], this[_stcache])
} catch (error) {
// if realpath fails, just provide a dummy error node
return new Node({
error,
path,
realpath: path,
parent,
root,
loadOverrides,
})
}
}
const cached = this.#cache.get(path)
let node
// missing edges get a dummy node, assign the parent and return it
if (cached && !cached.dummy) {
cached.parent = parent
return cached
} else {
const params = {
installLinks: this.installLinks,
legacyPeerDeps: this.legacyPeerDeps,
path,
realpath: real,
parent,
root,
loadOverrides,
}
try {
const { content: pkg } = await PackageJson.normalize(real)
params.pkg = pkg
if (useRootOverrides && root.overrides) {
params.overrides = root.overrides.getNodeRule({ name: pkg.name, version: pkg.version })
}
} catch (err) {
if (err.code === 'EJSONPARSE') {
// TODO @npmcli/package-json should be doing this
err.path = join(real, 'package.json')
}
params.error = err
}
// soldier on if read-package-json raises an error, passing it to the
// Node which will attach it to its errors array (Link passes it along to
// its target node)
if (normalize(path) === real) {
node = this.#newNode(params)
} else {
node = await this.#newLink(params)
}
}
this.#cache.set(path, node)
return node
}
#newNode (options) {
// check it for an fsParent if it's a tree top. there's a decent chance
// it'll get parented later, making the fsParent scan a no-op, but better
// safe than sorry, since it's cheap.
const { parent, realpath } = options
if (!parent) {
this.#topNodes.add(realpath)
}
return new Node(options)
}
async #newLink (options) {
const { realpath } = options
this.#topNodes.add(realpath)
const target = this.#cache.get(realpath)
const link = new Link({ ...options, target })
if (!target) {
// Link set its target itself in this case
this.#cache.set(realpath, link.target)
// if a link target points at a node outside of the root tree's
// node_modules hierarchy, then load that node as well.
await this.#loadFSTree(link.target)
}
return link
}
async #loadFSTree (node) {
const did = this.#actualTreeLoaded
if (!node.isLink && !did.has(node.target.realpath)) {
did.add(node.target.realpath)
await this.#loadFSChildren(node.target)
return Promise.all(
[...node.target.children.entries()]
.filter(([, kid]) => !did.has(kid.realpath))
.map(([, kid]) => this.#loadFSTree(kid))
)
}
}
// create child nodes for all the entries in node_modules
// and attach them to the node as a parent
async #loadFSChildren (node) {
const nm = resolve(node.realpath, 'node_modules')
try {
const kids = await readdirScoped(nm).then(paths => paths.map(p => p.replace(/\\/g, '/')))
return Promise.all(
// ignore . dirs and retired scoped package folders
kids.filter(kid => !/^(@[^/]+\/)?\./.test(kid))
.filter(kid => this.#filter(node, kid))
.map(kid => this.#loadFSNode({
parent: node,
path: resolve(nm, kid),
})))
} catch {
// error in the readdir is not fatal, just means no kids
}
}
async #findMissingEdges () {
// try to resolve any missing edges by walking up the directory tree,
// checking for the package in each node_modules folder. stop at the
// root directory.
// The tricky move here is that we load a "dummy" node for the folder
// containing the node_modules folder, so that it can be assigned as
// the fsParent. It's a bad idea to *actually* load that full node,
// because people sometimes develop in ~/projects/node_modules/...
// so we'd end up loading a massive tree with lots of unrelated junk.
const nmContents = new Map()
const tree = this.#actualTree
for (const node of tree.inventory.values()) {
const ancestor = commonAncestorPath(node.realpath, this.path)
const depPromises = []
for (const [name, edge] of node.edgesOut.entries()) {
const notMissing = !edge.missing &&
!(edge.to && (edge.to.dummy || edge.to.parent !== node))
if (notMissing) {
continue
}
// start the walk from the dirname, because we would have found
// the dep in the loadFSTree step already if it was local.
for (const p of walkUp(dirname(node.realpath))) {
// only walk as far as the nearest ancestor
// this keeps us from going into completely unrelated
// places when a project is just missing something, but
// allows for finding the transitive deps of link targets.
// ie, if it has to go up and back out to get to the path
// from the nearest common ancestor, we've gone too far.
if (ancestor && /^\.\.(?:[\\/]|$)/.test(relative(ancestor, p))) {
break
}
let entries
if (!nmContents.has(p)) {
entries = await readdirScoped(p + '/node_modules')
.catch(() => []).then(paths => paths.map(p => p.replace(/\\/g, '/')))
nmContents.set(p, entries)
} else {
entries = nmContents.get(p)
}
if (!entries.includes(name)) {
continue
}
let d
if (!this.#cache.has(p)) {
d = new Node({ path: p, root: node.root, dummy: true })
this.#cache.set(p, d)
} else {
d = this.#cache.get(p)
}
if (d.dummy) {
// it's a placeholder, so likely would not have loaded this dep,
// unless another dep in the tree also needs it.
const depPath = normalize(`${p}/node_modules/${name}`)
const cached = this.#cache.get(depPath)
if (!cached || cached.dummy) {
depPromises.push(this.#loadFSNode({
path: depPath,
root: node.root,
parent: d,
}).then(node => this.#loadFSTree(node)))
}
}
break
}
}
await Promise.all(depPromises)
}
}
}
@@ -0,0 +1,277 @@
const { resolve } = require('node:path')
// mixin providing the loadVirtual method
const mapWorkspaces = require('@npmcli/map-workspaces')
const PackageJson = require('@npmcli/package-json')
const nameFromFolder = require('@npmcli/name-from-folder')
const consistentResolve = require('../consistent-resolve.js')
const Shrinkwrap = require('../shrinkwrap.js')
const Node = require('../node.js')
const Link = require('../link.js')
const relpath = require('../relpath.js')
const calcDepFlags = require('../calc-dep-flags.js')
const treeCheck = require('../tree-check.js')
const flagsSuspect = Symbol.for('flagsSuspect')
const setWorkspaces = Symbol.for('setWorkspaces')
module.exports = cls => class VirtualLoader extends cls {
#rootOptionProvided
// public method
async loadVirtual (options = {}) {
if (this.virtualTree) {
return this.virtualTree
}
// allow the user to set reify options on the ctor as well.
// XXX: deprecate separate reify() options object.
options = { ...this.options, ...options }
if (options.root && options.root.meta) {
await this.#loadFromShrinkwrap(options.root.meta, options.root)
return treeCheck(this.virtualTree)
}
const s = await Shrinkwrap.load({
path: this.path,
lockfileVersion: this.options.lockfileVersion,
resolveOptions: this.options,
})
if (!s.loadedFromDisk && !options.root) {
const er = new Error('loadVirtual requires existing shrinkwrap file')
throw Object.assign(er, { code: 'ENOLOCK' })
}
// when building the ideal tree, we pass in a root node to this function
// otherwise, load it from the root package json or the lockfile
const pkg = await PackageJson.normalize(this.path).then(p => p.content).catch(() => s.data.packages[''] || {})
// TODO clean this up
const {
root = await this[setWorkspaces](this.#loadNode('', pkg, true)),
} = options
this.#rootOptionProvided = options.root
await this.#loadFromShrinkwrap(s, root)
root.assertRootOverrides()
return treeCheck(this.virtualTree)
}
async #loadFromShrinkwrap (s, root) {
if (!this.#rootOptionProvided) {
// root is never any of these things, but might be a brand new
// baby Node object that never had its dep flags calculated.
root.unsetDepFlags()
} else {
this[flagsSuspect] = true
}
this.#checkRootEdges(s, root)
root.meta = s
this.virtualTree = root
// separate out link metadata, and create Node objects for nodes
const links = new Map()
const nodes = new Map([['', root]])
for (const [location, meta] of Object.entries(s.data.packages)) {
// skip the root because we already got it
if (!location) {
continue
}
if (meta.link) {
links.set(location, meta)
} else {
nodes.set(location, this.#loadNode(location, meta))
}
}
await this.#resolveLinks(links, nodes)
if (!(s.originalLockfileVersion >= 2)) {
this.#assignBundles(nodes)
}
if (this[flagsSuspect]) {
// reset all dep flags
// can't use inventory here, because virtualTree might not be root
for (const node of nodes.values()) {
if (node.isRoot || node === this.#rootOptionProvided) {
continue
}
node.resetDepFlags()
}
calcDepFlags(this.virtualTree, !this.#rootOptionProvided)
}
return root
}
// check the lockfile deps, and see if they match. if they do not
// then we have to reset dep flags at the end. for example, if the
// user manually edits their package.json file, then we need to know
// that the idealTree is no longer entirely trustworthy.
#checkRootEdges (s, root) {
// loaded virtually from tree, no chance of being out of sync
// ancient lockfiles are critically damaged by this process,
// so we need to just hope for the best in those cases.
if (!s.loadedFromDisk || s.ancientLockfile) {
return
}
const lock = s.get('')
const prod = lock.dependencies || {}
const dev = lock.devDependencies || {}
const optional = lock.optionalDependencies || {}
const peer = lock.peerDependencies || {}
const peerOptional = {}
if (lock.peerDependenciesMeta) {
for (const [name, meta] of Object.entries(lock.peerDependenciesMeta)) {
if (meta.optional && peer[name] !== undefined) {
peerOptional[name] = peer[name]
delete peer[name]
}
}
}
for (const name of Object.keys(optional)) {
delete prod[name]
}
const lockWS = {}
const workspaces = mapWorkspaces.virtual({
cwd: this.path,
lockfile: s.data,
})
for (const [name, path] of workspaces.entries()) {
lockWS[name] = `file:${path}`
}
// Should rootNames exclude optional?
const rootNames = new Set(root.edgesOut.keys())
const lockByType = ({ dev, optional, peer, peerOptional, prod, workspace: lockWS })
// Find anything in shrinkwrap deps that doesn't match root's type or spec
for (const type in lockByType) {
const deps = lockByType[type]
for (const name in deps) {
const edge = root.edgesOut.get(name)
if (!edge || edge.type !== type || edge.spec !== deps[name]) {
return this[flagsSuspect] = true
}
rootNames.delete(name)
}
}
// Something was in root that's not accounted for in shrinkwrap
if (rootNames.size) {
return this[flagsSuspect] = true
}
}
// links is the set of metadata, and nodes is the map of non-Link nodes
// Set the targets to nodes in the set, if we have them (we might not)
// XXX build-ideal-tree also has a #resolveLinks, is there overlap?
async #resolveLinks (links, nodes) {
for (const [location, meta] of links.entries()) {
const targetPath = resolve(this.path, meta.resolved)
const targetLoc = relpath(this.path, targetPath)
const target = nodes.get(targetLoc)
if (!target) {
const err = new Error(
`Missing target in lock file: "${targetLoc}" is referenced by "${location}" but does not exist.
To fix:
1. rm package-lock.json
2. npm install`
)
err.code = 'EMISSINGTARGET'
throw err
}
const link = this.#loadLink(location, targetLoc, target, meta)
nodes.set(location, link)
nodes.set(targetLoc, link.target)
// we always need to read the package.json for link targets
// outside node_modules because they can be changed by the local user
if (!link.target.parent) {
await PackageJson.normalize(link.realpath).then(p => link.target.package = p.content).catch(() => null)
}
}
}
#assignBundles (nodes) {
for (const [location, node] of nodes) {
// Skip assignment of parentage for the root package
if (!location || node.isLink && !node.target.location) {
continue
}
const { name, parent, package: { inBundle } } = node
if (!parent) {
continue
}
// read inBundle from package because 'package' here is
// actually a v2 lockfile metadata entry.
// If the *parent* is also bundled, though, or if the parent has
// no dependency on it, then we assume that it's being pulled in
// just by virtue of its parent or a transitive dep being bundled.
const { package: ppkg } = parent
const { inBundle: parentBundled } = ppkg
if (inBundle && !parentBundled && parent.edgesOut.has(node.name)) {
if (!ppkg.bundleDependencies) {
ppkg.bundleDependencies = [name]
} else {
ppkg.bundleDependencies.push(name)
}
}
}
}
#loadNode (location, sw, loadOverrides) {
const p = this.virtualTree ? this.virtualTree.realpath : this.path
const path = resolve(p, location)
// shrinkwrap doesn't include package name unless necessary
if (!sw.name) {
sw.name = nameFromFolder(path)
}
const node = new Node({
installLinks: this.installLinks,
legacyPeerDeps: this.legacyPeerDeps,
root: this.virtualTree,
path,
realpath: path,
integrity: sw.integrity,
resolved: consistentResolve(sw.resolved, this.path, path),
pkg: sw,
hasShrinkwrap: sw.hasShrinkwrap,
loadOverrides,
// cast to boolean because they're undefined in the lock file when false
extraneous: !!sw.extraneous,
devOptional: !!(sw.devOptional || sw.dev || sw.optional),
peer: !!sw.peer,
optional: !!sw.optional,
dev: !!sw.dev,
})
return node
}
#loadLink (location, targetLoc, target) {
const path = resolve(this.path, location)
const link = new Link({
installLinks: this.installLinks,
legacyPeerDeps: this.legacyPeerDeps,
path,
realpath: resolve(this.path, targetLoc),
target,
pkg: target && target.package,
})
link.extraneous = target.extraneous
link.devOptional = target.devOptional
link.peer = target.peer
link.optional = target.optional
link.dev = target.dev
return link
}
}
@@ -0,0 +1,419 @@
// Arborist.rebuild({path = this.path}) will do all the binlinks and
// bundle building needed. Called by reify, and by `npm rebuild`.
const PackageJson = require('@npmcli/package-json')
const binLinks = require('bin-links')
const localeCompare = require('@isaacs/string-locale-compare')('en')
const promiseAllRejectLate = require('promise-all-reject-late')
const runScript = require('@npmcli/run-script')
const { callLimit: promiseCallLimit } = require('promise-call-limit')
const { depth: dfwalk } = require('treeverse')
const { isNodeGypPackage, defaultGypInstallScript } = require('@npmcli/node-gyp')
const { promiseRetry } = require('@gar/promise-retry')
const { log, time } = require('proc-log')
const { resolve } = require('node:path')
const { isScriptAllowed } = require('../script-allowed.js')
const boolEnv = b => b ? '1' : ''
const sortNodes = (a, b) => (a.depth - b.depth) || localeCompare(a.path, b.path)
const _checkBins = Symbol.for('checkBins')
// defined by reify mixin
const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
const _trashList = Symbol.for('trashList')
module.exports = cls => class Builder extends cls {
#doHandleOptionalFailure
#oldMeta = null
#queues = {
preinstall: [],
install: [],
postinstall: [],
prepare: [],
bin: [],
}
async rebuild ({ nodes, handleOptionalFailure = false } = {}) {
// nothing to do if we're not building anything!
if (this.options.ignoreScripts && !this.options.binLinks) {
return
}
// when building for the first time, as part of reify, we ignore
// failures in optional nodes, and just delete them. however, when
// running JUST a rebuild, we treat optional failures as real fails
this.#doHandleOptionalFailure = handleOptionalFailure
if (!nodes) {
nodes = await this.#loadDefaultNodes()
}
// separates links nodes so that it can run
// prepare scripts and link bins in the expected order
const timeEnd = time.start('build')
const {
depNodes,
linkNodes,
} = this.#retrieveNodesByType(nodes)
// build regular deps
await this.#build(depNodes, {})
// build link deps
if (linkNodes.size) {
this.#queues = {
preinstall: [],
install: [],
postinstall: [],
prepare: [],
bin: [],
}
await this.#build(linkNodes, { type: 'links' })
}
timeEnd()
}
// if we don't have a set of nodes, then just rebuild
// the actual tree on disk.
async #loadDefaultNodes () {
let nodes
const tree = await this.loadActual()
let filterSet
if (!this.options.workspacesEnabled) {
filterSet = this.excludeWorkspacesDependencySet(tree)
nodes = tree.inventory.filter(node =>
filterSet.has(node) || node.isProjectRoot
)
} else if (this.options.workspaces.length) {
filterSet = this.workspaceDependencySet(
tree,
this.options.workspaces,
this.options.includeWorkspaceRoot
)
nodes = tree.inventory.filter(node => filterSet.has(node))
} else {
nodes = tree.inventory.values()
}
return nodes
}
#retrieveNodesByType (nodes) {
const depNodes = new Set()
const linkNodes = new Set()
const storeNodes = new Set()
for (const node of nodes) {
if (node.isStoreLink) {
storeNodes.add(node)
} else if (node.isLink) {
linkNodes.add(node)
} else {
depNodes.add(node)
}
}
// Make sure that store linked nodes are processed last.
// We can't process store links separately or else lifecycle scripts on
// standard nodes might not have bin links yet.
for (const node of storeNodes) {
depNodes.add(node)
}
// deduplicates link nodes and their targets, avoids
// calling lifecycle scripts twice when running `npm rebuild`
// ref: https://github.com/npm/cli/issues/2905
//
// we avoid doing so if global=true since `bin-links` relies
// on having the target nodes available in global mode.
if (!this.options.global) {
for (const node of linkNodes) {
depNodes.delete(node.target)
}
}
return {
depNodes,
linkNodes,
}
}
async #build (nodes, { type = 'deps' }) {
const timeEnd = time.start(`build:${type}`)
await this.#buildQueues(nodes)
if (!this.options.ignoreScripts) {
await this.#runScripts('preinstall')
}
// links should run prepare scripts and only link bins after that
if (type === 'links') {
if (!this.options.ignoreScripts) {
await this.#runScripts('prepare')
}
}
if (this.options.binLinks) {
await this.#linkAllBins()
}
if (!this.options.ignoreScripts) {
await this.#runScripts('install')
await this.#runScripts('postinstall')
}
timeEnd()
}
async #buildQueues (nodes) {
const timeEnd = time.start('build:queue')
const set = new Set()
const promises = []
for (const node of nodes) {
promises.push(this.#addToBuildSet(node, set))
// if it has bundle deps, add those too, if rebuildBundle
if (this.options.rebuildBundle !== false) {
const bd = node.package.bundleDependencies
if (bd && bd.length) {
dfwalk({
tree: node,
leave: node => promises.push(this.#addToBuildSet(node, set)),
getChildren: node => [...node.children.values()],
filter: node => node.inBundle,
})
}
}
}
await promiseAllRejectLate(promises)
// now sort into the queues for the 4 things we have to do
// run in the same predictable order that buildIdealTree uses
// there's no particular reason for doing it in this order rather
// than another, but sorting *somehow* makes it consistent.
const queue = [...set].sort(sortNodes)
for (const node of queue) {
const { package: { bin, scripts = {} } } = node.target
const { preinstall, install, postinstall, prepare } = scripts
const tests = { bin, preinstall, install, postinstall, prepare }
for (const [key, has] of Object.entries(tests)) {
if (has) {
this.#queues[key].push(node)
}
}
}
timeEnd()
}
async [_checkBins] (node) {
// if the node is a global top, and we're not in force mode, then
// any existing bins need to either be missing, or a symlink into
// the node path. Otherwise a package can have a preinstall script
// that unlinks something, to allow them to silently overwrite system
// binaries, which is unsafe and insecure.
if (!node.globalTop || this.options.force) {
return
}
const { path, package: pkg } = node
await binLinks.checkBins({ pkg, path, top: true, global: true })
}
async #addToBuildSet (node, set, refreshed = false) {
if (set.has(node)) {
return
}
// Phase 1 allowScripts gate: a `false` verdict from the policy matcher
// means the user explicitly denied install scripts for this node, so skip
// it. `true` and `null` (unreviewed) both fall through to the existing
// detection logic — unreviewed nodes still run their scripts in Phase 1
// and are surfaced via the post-reify advisory warning. The global
// --ignore-scripts kill switch in #build() still takes precedence, and
// --dangerously-allow-all-scripts bypasses this gate entirely.
if (!this.options.dangerouslyAllowAllScripts &&
isScriptAllowed(node, this.options.allowScripts) === false) {
return
}
if (this.#oldMeta === null) {
const { root: { meta } } = node
this.#oldMeta = meta && meta.loadedFromDisk &&
!(meta.originalLockfileVersion >= 2)
}
const { package: pkg, hasInstallScript } = node.target
const { gypfile, bin, scripts = {} } = pkg
const { preinstall, install, postinstall, prepare } = scripts
const anyScript = preinstall || install || postinstall || prepare
if (!refreshed && !anyScript && (hasInstallScript || this.#oldMeta)) {
// we either have an old metadata (and thus might have scripts)
// or we have an indication that there's install scripts (but
// don't yet know what they are) so we have to load the package.json
// from disk to see what the deal is. Failure here just means
// no scripts to add, probably borked package.json.
// add to the set then remove while we're reading the pj, so we
// don't accidentally hit it multiple times.
set.add(node)
const { content: pkg } = await PackageJson.normalize(node.path).catch(() => {
return { content: {} }
})
set.delete(node)
const { scripts = {} } = pkg
node.package.scripts = scripts
return this.#addToBuildSet(node, set, true)
}
// Rebuild node-gyp dependencies lacking an install or preinstall script
// note that 'scripts' might be missing entirely, and the package may
// set gypfile:false to avoid this automatic detection.
const isGyp = gypfile !== false &&
!install &&
!preinstall &&
await isNodeGypPackage(node.path)
if (bin || preinstall || install || postinstall || prepare || isGyp) {
if (bin) {
await this[_checkBins](node)
}
if (isGyp) {
scripts.install = defaultGypInstallScript
node.package.scripts = scripts
}
set.add(node)
}
}
async #runScripts (event) {
const queue = this.#queues[event]
if (!queue.length) {
return
}
const timeEnd = time.start(`build:run:${event}`)
const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe'
const limit = this.options.foregroundScripts ? 1 : undefined
await promiseCallLimit(queue.map(node => async () => {
const {
path,
integrity,
resolved,
optional,
peer,
dev,
devOptional,
package: pkg,
location,
} = node.target
// skip any that we know we'll be deleting
// or links to store entries (their scripts run on the store
// entry itself, not through the link)
if (this[_trashList].has(path) || (node.isLink && node.target?.isInStore)) {
return
}
const timeEndLocation = time.start(`build:run:${event}:${location}`)
log.info('run', pkg._id, event, location, pkg.scripts[event])
const env = {
npm_package_resolved: resolved,
npm_package_integrity: integrity,
npm_package_json: resolve(path, 'package.json'),
npm_package_optional: boolEnv(optional),
npm_package_dev: boolEnv(dev),
npm_package_peer: boolEnv(peer),
npm_package_dev_optional:
boolEnv(devOptional && !dev && !optional),
}
const runOpts = {
event,
path,
pkg,
stdio,
env,
scriptShell: this.options.scriptShell,
}
const p = runScript(runOpts).catch(er => {
const { code, signal } = er
log.info('run', pkg._id, event, { code, signal })
throw er
}).then(({ args, code, signal, stdout, stderr }) => {
this.scriptsRun.add({
pkg,
path,
event,
// I do not know why this needs to be on THIS line but refactoring
// this function would be quite a process
// eslint-disable-next-line promise/always-return
cmd: args && args[args.length - 1],
env,
code,
signal,
stdout,
stderr,
})
log.info('run', pkg._id, event, { code, signal })
})
await (this.#doHandleOptionalFailure
? this[_handleOptionalFailure](node, p)
: p)
timeEndLocation()
}), { limit })
timeEnd()
}
async #linkAllBins () {
const queue = this.#queues.bin
if (!queue.length) {
return
}
const timeEnd = time.start('build:link')
const promises = []
// sort the queue by node path, so that the module-local collision
// detector in bin-links will always resolve the same way.
for (const node of queue.sort(sortNodes)) {
// TODO these run before they're awaited
promises.push(this.#createBinLinks(node))
}
await promiseAllRejectLate(promises)
timeEnd()
}
async #createBinLinks (node) {
if (this[_trashList].has(node.path)) {
return
}
const timeEnd = time.start(`build:link:${node.location}`)
// On Windows, antivirus/indexer can transiently lock files, causing EPERM/EACCES/EBUSY on the rename inside write-file-atomic (used by bin-links/fix-bin.js), so, retry with backoff.
const p = promiseRetry((retry) => binLinks({
pkg: node.package,
path: node.path,
top: !!(node.isTop || node.globalTop),
force: this.options.force,
global: !!node.globalTop,
}).catch(/* istanbul ignore next - Windows-only transient antivirus locks */ err => {
if (process.platform === 'win32' &&
(err.code === 'EPERM' || err.code === 'EACCES' || err.code === 'EBUSY')) {
return retry(err)
}
throw err
}), { retries: 5, minTimeout: 500 })
await (this.#doHandleOptionalFailure
? this[_handleOptionalFailure](node, p)
: p)
timeEnd()
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,330 @@
// an object representing the set of vulnerabilities in a tree
const localeCompare = require('@isaacs/string-locale-compare')('en')
const npa = require('npm-package-arg')
const pickManifest = require('npm-pick-manifest')
const Vuln = require('./vuln.js')
const Calculator = require('@npmcli/metavuln-calculator')
const { log, time } = require('proc-log')
const npmFetch = require('npm-registry-fetch')
class AuditReport extends Map {
#omit
error = null
topVulns = new Map()
static load (tree, opts) {
return new AuditReport(tree, opts).run()
}
get auditReportVersion () {
return 2
}
toJSON () {
const obj = {
auditReportVersion: this.auditReportVersion,
vulnerabilities: {},
metadata: {
vulnerabilities: {
info: 0,
low: 0,
moderate: 0,
high: 0,
critical: 0,
total: this.size,
},
dependencies: {
prod: 0,
dev: 0,
optional: 0,
peer: 0,
peerOptional: 0,
total: this.tree.inventory.size - 1,
},
},
}
for (const node of this.tree.inventory.values()) {
const { dependencies } = obj.metadata
let prod = true
for (const type of [
'dev',
'optional',
'peer',
'peerOptional',
]) {
if (node[type]) {
dependencies[type]++
prod = false
}
}
if (prod) {
dependencies.prod++
}
}
// if it doesn't have any topVulns, then it's fixable with audit fix
// for each topVuln, figure out if it's fixable with audit fix --force,
// or if we have to just delete the thing, and if the fix --force will
// require a semver major update.
const vulnerabilities = []
for (const [name, vuln] of this.entries()) {
vulnerabilities.push([name, vuln.toJSON()])
obj.metadata.vulnerabilities[vuln.severity]++
}
obj.vulnerabilities = vulnerabilities
.sort(([a], [b]) => localeCompare(a, b))
.reduce((set, [name, vuln]) => {
set[name] = vuln
return set
}, {})
return obj
}
constructor (tree, opts = {}) {
super()
this.#omit = new Set(opts.omit || [])
this.calculator = new Calculator(opts)
this.options = opts
this.tree = tree
this.filterSet = opts.filterSet
}
async run () {
this.report = await this.#getReport()
log.silly('audit report', this.report)
if (this.report) {
await this.#init()
}
return this
}
isVulnerable (node) {
const vuln = this.get(node.packageName)
return !!(vuln && vuln.isVulnerable(node))
}
async #init () {
const timeEnd = time.start('auditReport:init')
const promises = []
for (const [name, advisories] of Object.entries(this.report)) {
for (const advisory of advisories) {
promises.push(this.calculator.calculate(name, advisory))
}
}
// now the advisories are calculated with a set of versions
// and the packument. turn them into our style of vuln objects
// which also have the affected nodes, and also create entries
// for all the metavulns that we find from dependents.
const advisories = new Set(await Promise.all(promises))
const seen = new Set()
for (const advisory of advisories) {
const { name, range } = advisory
const k = `${name}@${range}`
const vuln = this.get(name) || new Vuln({ name, advisory })
if (this.has(name)) {
vuln.addAdvisory(advisory)
}
super.set(name, vuln)
// don't flag the exact same name/range more than once
// adding multiple advisories with the same range is fine, but no
// need to search for nodes we already would have added.
if (!seen.has(k)) {
const p = []
for (const node of this.tree.inventory.query('packageName', name)) {
if (!this.shouldAudit(node)) {
continue
}
// if not vulnerable by this advisory, keep searching
if (!advisory.testVersion(node.version)) {
continue
}
// we will have loaded the source already if this is a metavuln
if (advisory.type === 'metavuln') {
vuln.addVia(this.get(advisory.dependency))
}
// already marked this one, no need to do it again
if (vuln.nodes.has(node)) {
continue
}
// haven't marked this one yet. get its dependents.
vuln.nodes.add(node)
for (const { from: dep, spec } of node.edgesIn) {
if (dep.isTop && !vuln.topNodes.has(dep)) {
vuln.fixAvailable = this.#fixAvailable(vuln, spec)
if (vuln.fixAvailable !== true) {
// now we know the top node is vulnerable, and cannot be
// upgraded out of the bad place without --force. But, there's
// no need to add it to the actual vulns list, because nothing
// depends on root.
this.topVulns.set(vuln.name, vuln)
vuln.topNodes.add(dep)
}
} else {
// calculate a metavuln, if necessary
const calc = this.calculator.calculate(dep.packageName, advisory)
// eslint-disable-next-line promise/always-return
p.push(calc.then(meta => {
// eslint-disable-next-line promise/always-return
if (meta.testVersion(dep.version, spec)) {
advisories.add(meta)
}
}))
}
}
}
await Promise.all(p)
seen.add(k)
}
// make sure we actually got something. if not, remove it
// this can happen if you are loading from a lockfile created by
// npm v5, since it lists the current version of all deps,
// rather than the range that is actually depended upon,
// or if using --omit with the older audit endpoint.
if (this.get(name).nodes.size === 0) {
this.delete(name)
continue
}
// if the vuln is valid, but THIS advisory doesn't apply to any of
// the nodes it references, then remove it from the advisory list.
// happens when using omit with old audit endpoint.
for (const advisory of vuln.advisories) {
const relevant = [...vuln.nodes]
.some(n => advisory.testVersion(n.version))
if (!relevant) {
vuln.deleteAdvisory(advisory)
}
}
}
timeEnd()
}
// given the spec, see if there is a fix available at all, and note whether or not it's a semver major fix or not (i.e. will need --force)
#fixAvailable (vuln, spec) {
// TODO we return true, false, OR an object here. this is probably a bad pattern.
if (!vuln.testSpec(spec)) {
return true
}
// even if we HAVE a packument, if we're looking for it somewhere other than the registry and we have something vulnerable then we're stuck with it.
const specObj = npa(spec)
if (!specObj.registry) {
return false
}
if (specObj.subSpec) {
spec = specObj.subSpec.rawSpec
}
// we don't provide fixes for top nodes other than root, but we still check to see if the node is fixable with a different version, and note if that is a semver major bump.
try {
const {
_isSemVerMajor: isSemVerMajor,
version,
name,
} = pickManifest(vuln.packument, spec, {
...this.options,
before: null,
avoid: vuln.range,
avoidStrict: true,
})
return { name, version, isSemVerMajor }
} catch (er) {
return false
}
}
set () {
throw new Error('do not call AuditReport.set() directly')
}
async #getReport () {
// if we're not auditing, just return false
if (this.options.audit === false || this.options.offline === true || this.tree.inventory.size === 1) {
return null
}
const timeEnd = time.start('auditReport:getReport')
try {
const body = this.prepareBulkData()
log.silly('audit', 'bulk request', body)
// no sense asking if we don't have anything to audit,
// we know it'll be empty
if (!Object.keys(body).length) {
return null
}
const res = await npmFetch('/-/npm/v1/security/advisories/bulk', {
...this.options,
registry: this.options.auditRegistry || this.options.registry,
method: 'POST',
gzip: true,
body,
})
return await res.json()
} catch (er) {
log.verbose('audit error', er)
log.silly('audit error', String(er.body))
this.error = er
return null
} finally {
timeEnd()
}
}
// return true if we should audit this one
shouldAudit (node) {
if (
!node.version ||
node.isRoot ||
node.isLink ||
node.linksIn?.size > 0 ||
(this.filterSet && this.filterSet?.size !== 0 && !this.filterSet?.has(node))
) {
return false
}
if (this.#omit.size === 0) {
return true
}
return !node.shouldOmit(this.#omit)
}
prepareBulkData () {
const payload = {}
for (const name of this.tree.inventory.query('packageName')) {
const set = new Set()
for (const node of this.tree.inventory.query('packageName', name)) {
if (!this.shouldAudit(node)) {
continue
}
set.add(node.version)
}
if (set.size) {
payload[name] = [...set]
}
}
return payload
}
}
module.exports = AuditReport
@@ -0,0 +1,103 @@
// Dep flag (dev, peer, etc.) calculation requires default or reset flags.
// Flags are true by default and are unset to false as we walk deps.
// We iterate outward edges looking for dep flags that can
// be unset based on the current nodes flags and edge type.
// Examples:
// - a non-optional node with a non-optional edge out, the edge node should not be optional
// - a non-peer node with a non-peer edge out, the edge node should not be peer
// If a node is changed, we add to the queue and continue until no more changes.
// Flags that remain after all this unsetting should be valid.
// Examples:
// - a node still flagged optional must only be reachable via optional edges
// - a node still flagged peer must only be reachable via peer edges
const calcDepFlags = (tree, resetRoot = true) => {
if (resetRoot) {
tree.unsetDepFlags()
}
const seen = new Set()
const queue = [tree]
let node
while (node = queue.pop()) {
seen.add(node)
// Unset extraneous from all parents to avoid removal of children.
if (!node.extraneous) {
for (let n = node.resolveParent; n?.extraneous; n = n.resolveParent) {
n.extraneous = false
}
}
// for links, map their hierarchy appropriately
if (node.isLink) {
// node.target can be null, we check to ensure it's not null before proceeding
if (node.target == null) {
continue
}
node.target.dev = node.dev
node.target.optional = node.optional
node.target.devOptional = node.devOptional
node.target.peer = node.peer
node.target.extraneous = node.extraneous
queue.push(node.target)
continue
}
for (const { peer, optional, dev, to } of node.edgesOut.values()) {
// if the dep is missing, then its flags are already maximally unset
if (!to) {
continue
}
let changed = false
// only optional peer dependencies should stay extraneous
if (to.extraneous && !node.extraneous && !(peer && optional)) {
to.extraneous = false
changed = true
}
if (to.dev && !node.dev && !dev) {
to.dev = false
changed = true
}
if (to.optional && !node.optional && !optional) {
to.optional = false
changed = true
}
// devOptional is the *overlap* of the dev and optional tree.
// A node may be depended on by separate dev and optional nodes.
// It SHOULD NOT be removed when pruning dev OR optional.
// It SHOULD be removed when pruning dev AND optional.
// We only unset here if a node is not dev AND not optional because
// if we did unset, it would prevent any overlap deeper in the tree.
// We correct this later by removing if dev OR optional is set.
if (to.devOptional && !node.devOptional && !node.dev && !node.optional && !dev && !optional) {
to.devOptional = false
changed = true
}
if (to.peer && !node.peer && !peer) {
to.peer = false
changed = true
}
if (changed) {
queue.push(to)
}
}
}
// Remove incorrect devOptional flags now that we have walked all deps.
seen.delete(tree)
for (const node of seen.values()) {
if (node.devOptional && (node.dev || node.optional)) {
node.devOptional = false
}
}
}
module.exports = calcDepFlags
@@ -0,0 +1,436 @@
// Internal methods used by buildIdealTree.
// Answer the question: "can I put this dep here?"
//
// IMPORTANT: *nothing* in this class should *ever* modify or mutate the tree
// at all. The contract here is strictly limited to read operations. We call
// this in the process of walking through the ideal tree checking many
// different potential placement targets for a given node. If a change is made
// to the tree along the way, that can cause serious problems!
//
// In order to enforce this restriction, in debug mode, canPlaceDep() will
// snapshot the tree at the start of the process, and then at the end, will
// verify that it still matches the snapshot, and throw an error if any changes
// occurred.
//
// The algorithm is roughly like this:
// - check the node itself:
// - if there is no version present, and no conflicting edges from target,
// OK, provided all peers can be placed at or above the target.
// - if the current version matches, KEEP
// - if there is an older version present, which can be replaced, then
// - if satisfying and preferDedupe? KEEP
// - else: REPLACE
// - if there is a newer version present, and preferDedupe, REPLACE
// - if the version present satisfies the edge, KEEP
// - else: CONFLICT
// - if the node is not in conflict, check each of its peers:
// - if the peer can be placed in the target, continue
// - else if the peer can be placed in a parent, and there is no other
// conflicting version shadowing it, continue
// - else CONFLICT
// - If the peers are not in conflict, return the original node's value
//
// An exception to this logic is that if the target is the deepest location
// that a node can be placed, and the conflicting node can be placed deeper,
// then we will return REPLACE rather than CONFLICT, and Arborist will queue
// the replaced node for resolution elsewhere.
const localeCompare = require('@isaacs/string-locale-compare')('en')
const semver = require('semver')
const debug = require('./debug.js')
const peerEntrySets = require('./peer-entry-sets.js')
const deepestNestingTarget = require('./deepest-nesting-target.js')
const CONFLICT = Symbol('CONFLICT')
const OK = Symbol('OK')
const REPLACE = Symbol('REPLACE')
const KEEP = Symbol('KEEP')
class CanPlaceDep {
// dep is a dep that we're trying to place. it should already live in
// a virtual tree where its peer set is loaded as children of the root.
// target is the actual place where we're trying to place this dep
// in a node_modules folder.
// edge is the edge that we're trying to satisfy with this placement.
// parent is the CanPlaceDep object of the entry node when placing a peer.
constructor (options) {
const {
dep,
target,
edge,
preferDedupe,
parent = null,
peerPath = [],
explicitRequest = false,
} = options
debug(() => {
if (!dep) {
throw new Error('no dep provided to CanPlaceDep')
}
if (!target) {
throw new Error('no target provided to CanPlaceDep')
}
if (!edge) {
throw new Error('no edge provided to CanPlaceDep')
}
this._treeSnapshot = JSON.stringify([...target.root.inventory.entries()]
.map(([loc, { packageName, version, resolved }]) => {
return [loc, packageName, version, resolved]
}).sort(([a], [b]) => localeCompare(a, b)))
})
// the result of whether we can place it or not
this.canPlace = null
// if peers conflict, but this one doesn't, then that is useful info
this.canPlaceSelf = null
this.dep = dep
this.target = target
this.edge = edge
this.explicitRequest = explicitRequest
// preventing cycles when we check peer sets
this.peerPath = peerPath
// we always prefer to dedupe peers, because they are trying
// a bit harder to be singletons.
this.preferDedupe = !!preferDedupe || edge.peer
this.parent = parent
this.children = []
this.isSource = target === this.peerSetSource
this.name = edge.name
this.current = target.children.get(this.name)
this.targetEdge = target.edgesOut.get(this.name)
this.conflicts = new Map()
// check if this dep was already subject to a peerDep override while
// building the peerSet.
this.edgeOverride = !dep.satisfies(edge)
this.canPlace = this.checkCanPlace()
if (!this.canPlaceSelf) {
this.canPlaceSelf = this.canPlace
}
debug(() => {
const treeSnapshot = JSON.stringify([...target.root.inventory.entries()]
.map(([loc, { packageName, version, resolved }]) => {
return [loc, packageName, version, resolved]
}).sort(([a], [b]) => localeCompare(a, b)))
/* istanbul ignore if */
if (this._treeSnapshot !== treeSnapshot) {
throw Object.assign(new Error('tree changed in CanPlaceDep'), {
expect: this._treeSnapshot,
actual: treeSnapshot,
})
}
})
}
checkCanPlace () {
const { target, targetEdge, current, dep } = this
// if the dep failed to load, we're going to fail the build or
// prune it out anyway, so just move forward placing/replacing it.
if (dep.errors.length) {
return current ? REPLACE : OK
}
// cannot place peers inside their dependents, except for tops
if (targetEdge && targetEdge.peer && !target.isTop) {
return CONFLICT
}
// skip this test if there's a current node, because we might be able
// to dedupe against it anyway
if (!current &&
targetEdge &&
!dep.satisfies(targetEdge) &&
targetEdge !== this.edge) {
return CONFLICT
}
return current ? this.checkCanPlaceCurrent() : this.checkCanPlaceNoCurrent()
}
// we know that the target has a dep by this name in its node_modules
// already. Can return KEEP, REPLACE, or CONFLICT.
checkCanPlaceCurrent () {
const { preferDedupe, explicitRequest, current, target, edge, dep } = this
if (dep.matches(current)) {
if (current.satisfies(edge) || this.edgeOverride) {
return explicitRequest ? REPLACE : KEEP
}
}
const { version: curVer } = current
const { version: newVer } = dep
const tryReplace = curVer && newVer && semver.gte(newVer, curVer)
if (tryReplace && dep.canReplace(current)) {
// It's extremely rare that a replaceable node would be a conflict, if
// the current one wasn't a conflict, but it is theoretically possible
// if peer deps are pinned. In that case we treat it like any other
// conflict, and keep trying.
const cpp = this.canPlacePeers(REPLACE)
if (cpp !== CONFLICT) {
return cpp
}
}
// ok, can't replace the current with new one, but maybe current is ok?
if (current.satisfies(edge) && (!explicitRequest || preferDedupe)) {
return KEEP
}
// if we prefer deduping, then try replacing newer with older
if (preferDedupe && !tryReplace && dep.canReplace(current)) {
const cpp = this.canPlacePeers(REPLACE)
if (cpp !== CONFLICT) {
return cpp
}
}
// Check for interesting cases!
// First, is this the deepest place that this thing can go, and NOT the
// deepest place where the conflicting dep can go? If so, replace it,
// and let it re-resolve deeper in the tree.
const myDeepest = this.deepestNestingTarget
// ok, i COULD be placed deeper, so leave the current one alone.
if (target !== myDeepest) {
return CONFLICT
}
// if we are not checking a peerDep, then we MUST place it here, in the
// target that has a non-peer dep on it.
if (!edge.peer && target === edge.from) {
return this.canPlacePeers(REPLACE)
}
// if we aren't placing a peer in a set, then we're done here.
// This is ignored because it SHOULD be redundant, as far as I can tell,
// with the deepest target and target===edge.from tests. But until we
// can prove that isn't possible, this condition is here for safety.
/* istanbul ignore if - allegedly impossible */
if (!this.parent && !edge.peer) {
return CONFLICT
}
// check the deps in the peer group for each edge into that peer group
// if ALL of them can be pushed deeper, or if it's ok to replace its
// members with the contents of the new peer group, then we're good.
let canReplace = true
for (const [entryEdge, currentPeers] of peerEntrySets(current)) {
if (entryEdge === this.edge || entryEdge === this.peerEntryEdge) {
continue
}
// First, see if it's ok to just replace the peerSet entirely.
// we do this by walking out from the entryEdge, because in a case like
// this:
//
// v -> PEER(a@1||2)
// a@1 -> PEER(b@1)
// a@2 -> PEER(b@2)
// b@1 -> PEER(a@1)
// b@2 -> PEER(a@2)
//
// root
// +-- v
// +-- a@2
// +-- b@2
//
// Trying to place a peer group of (a@1, b@1) would fail to note that
// they can be replaced, if we did it by looping 1 by 1. If we are
// replacing something, we don't have to check its peer deps, because
// the peerDeps in the placed peerSet will presumably satisfy.
const entryNode = entryEdge.to
const entryRep = dep.parent.children.get(entryNode.name)
if (entryRep) {
if (entryRep.canReplace(entryNode, dep.parent.children.keys())) {
continue
}
}
let canClobber = !entryRep
if (!entryRep) {
const peerReplacementWalk = new Set([entryNode])
OUTER: for (const currentPeer of peerReplacementWalk) {
for (const edge of currentPeer.edgesOut.values()) {
if (!edge.peer || !edge.valid) {
continue
}
const rep = dep.parent.children.get(edge.name)
if (!rep) {
if (edge.to) {
peerReplacementWalk.add(edge.to)
}
continue
}
if (!rep.satisfies(edge)) {
canClobber = false
break OUTER
}
}
}
}
if (canClobber) {
continue
}
// ok, we can't replace, but maybe we can nest the current set deeper?
let canNestCurrent = true
for (const currentPeer of currentPeers) {
if (!canNestCurrent) {
break
}
// still possible to nest this peerSet
const curDeep = deepestNestingTarget(entryEdge.from, currentPeer.name)
if (curDeep === target || target.isDescendantOf(curDeep)) {
canNestCurrent = false
canReplace = false
}
if (canNestCurrent) {
continue
}
}
}
// if we can nest or replace all the current peer groups, we can replace.
if (canReplace) {
return this.canPlacePeers(REPLACE)
}
return CONFLICT
}
checkCanPlaceNoCurrent () {
const { target, peerEntryEdge, dep, name } = this
// check to see what that name resolves to here, and who may depend on
// being able to reach it by crawling up past the parent. we know
// that it's not the target's direct child node, and if it was a direct
// dep of the target, we would have conflicted earlier.
const current = target !== peerEntryEdge.from && target.resolve(name)
if (current) {
for (const edge of current.edgesIn.values()) {
if (edge.from.isDescendantOf(target) && edge.valid) {
if (!dep.satisfies(edge)) {
return CONFLICT
}
}
}
}
// no objections, so this is fine as long as peers are ok here.
return this.canPlacePeers(OK)
}
get deepestNestingTarget () {
const start = this.parent ? this.parent.deepestNestingTarget
: this.edge.from
return deepestNestingTarget(start, this.name)
}
get conflictChildren () {
return this.allChildren.filter(c => c.canPlace === CONFLICT)
}
get allChildren () {
const set = new Set(this.children)
for (const child of set) {
for (const grandchild of child.children) {
set.add(grandchild)
}
}
return [...set]
}
get top () {
return this.parent ? this.parent.top : this
}
// check if peers can go here. returns state or CONFLICT
canPlacePeers (state) {
this.canPlaceSelf = state
if (this._canPlacePeers) {
return this._canPlacePeers
}
// TODO: represent peerPath in ERESOLVE error somehow?
const peerPath = [...this.peerPath, this.dep]
let sawConflict = false
for (const peerEdge of this.dep.edgesOut.values()) {
if (!peerEdge.peer || !peerEdge.to || peerPath.includes(peerEdge.to)) {
continue
}
const peer = peerEdge.to
// it may be the case that the *initial* dep can be nested, but a peer
// of that dep needs to be placed shallower, because the target has
// a peer dep on the peer as well.
const target = deepestNestingTarget(this.target, peer.name)
const cpp = new CanPlaceDep({
dep: peer,
target,
parent: this,
edge: peerEdge,
peerPath,
// always place peers in preferDedupe mode
preferDedupe: true,
})
/* istanbul ignore next */
debug(() => {
if (this.children.some(c => c.dep === cpp.dep)) {
throw new Error('checking same dep repeatedly')
}
})
this.children.push(cpp)
if (cpp.canPlace === CONFLICT) {
sawConflict = true
}
}
this._canPlacePeers = sawConflict ? CONFLICT : state
return this._canPlacePeers
}
// what is the node that is causing this peerSet to be placed?
get peerSetSource () {
return this.parent ? this.parent.peerSetSource : this.edge.from
}
get peerEntryEdge () {
return this.top.edge
}
static get CONFLICT () {
return CONFLICT
}
static get OK () {
return OK
}
static get REPLACE () {
return REPLACE
}
static get KEEP () {
return KEEP
}
get description () {
const { canPlace } = this
return canPlace && canPlace.description ||
/* istanbul ignore next - old node affordance */ canPlace
}
}
module.exports = CanPlaceDep
@@ -0,0 +1,50 @@
// package children are represented with a Map object, but many file systems
// are case-insensitive and unicode-normalizing, so we need to treat
// node.children.get('FOO') and node.children.get('foo') as the same thing.
module.exports = class CIMap extends Map {
#keys = new Map()
constructor (items = []) {
super()
for (const [key, val] of items) {
this.set(key, val)
}
}
#normKey (key) {
if (typeof key !== 'string') {
return key
}
return key.normalize('NFKD').toLowerCase()
}
get (key) {
const normKey = this.#normKey(key)
return this.#keys.has(normKey) ? super.get(this.#keys.get(normKey))
: undefined
}
set (key, val) {
const normKey = this.#normKey(key)
if (this.#keys.has(normKey)) {
super.delete(this.#keys.get(normKey))
}
this.#keys.set(normKey, key)
return super.set(key, val)
}
delete (key) {
const normKey = this.#normKey(key)
if (this.#keys.has(normKey)) {
const prevKey = this.#keys.get(normKey)
this.#keys.delete(normKey)
return super.delete(prevKey)
}
}
has (key) {
const normKey = this.#normKey(key)
return this.#keys.has(normKey) && super.has(this.#keys.get(normKey))
}
}
@@ -0,0 +1,44 @@
// take a path and a resolved value, and turn it into a resolution from
// the given new path. This is used with converting a package.json's
// relative file: path into one suitable for a lockfile, or between
// lockfiles, and for converting hosted git repos to a consistent url type.
const npa = require('npm-package-arg')
const relpath = require('./relpath.js')
const consistentResolve = (resolved, fromPath, toPath, relPaths = false) => {
if (!resolved) {
return null
}
try {
const hostedOpt = { noCommittish: false }
const {
fetchSpec,
saveSpec,
type,
hosted,
rawSpec,
raw,
} = npa(resolved, fromPath)
if (type === 'file' || type === 'directory') {
if (relPaths && toPath) {
return `file:${relpath(toPath, fetchSpec)}`
}
return `file:${fetchSpec}`
}
if (hosted) {
return `git+${hosted.auth ? hosted.https(hostedOpt) : hosted.sshurl(hostedOpt)}`
}
if (type === 'git') {
return saveSpec
}
if (rawSpec === '*') {
return raw
}
return rawSpec
} catch (_) {
// whatever we passed in was not acceptable to npa.
// leave it 100% untouched.
return resolved
}
}
module.exports = consistentResolve
@@ -0,0 +1,32 @@
// certain assertions we should do only when testing arborist itself, because
// they are too expensive or aggressive and would break user programs if we
// miss a situation where they are actually valid.
//
// call like this:
//
// /* istanbul ignore next - debug check */
// debug(() => {
// if (someExpensiveCheck)
// throw new Error('expensive check should have returned false')
// })
// run in debug mode if explicitly requested, running arborist tests,
// or working in the arborist project directory.
const debug = process.env.ARBORIST_DEBUG !== '0' && (
process.env.ARBORIST_DEBUG === '1' ||
/\barborist\b/.test(process.env.NODE_DEBUG || '') ||
process.env.npm_package_name === '@npmcli/arborist' &&
['test', 'snap'].includes(process.env.npm_lifecycle_event) ||
process.cwd() === require('node:path').resolve(__dirname, '..')
)
module.exports = debug ? fn => fn() : () => {}
const red = process.stderr.isTTY ? msg => `\x1B[31m${msg}\x1B[39m` : m => m
module.exports.log = (...msg) => module.exports(() => {
const { format } = require('node:util')
const prefix = `\n${process.pid} ${red(format(msg.shift()))} `
msg = (prefix + format(...msg).trim().split('\n').join(prefix)).trim()
/* eslint-disable-next-line no-console */
console.error(msg)
})
@@ -0,0 +1,18 @@
// given a starting node, what is the *deepest* target where name could go?
// This is not on the Node class for the simple reason that we sometimes
// need to check the deepest *potential* target for a Node that is not yet
// added to the tree where we are checking.
const deepestNestingTarget = (start, name) => {
for (const target of start.ancestry()) {
// note: this will skip past the first target if edge is peer
if (target.isProjectRoot || !target.resolveParent || target.globalTop) {
return target
}
const targetEdge = target.edgesOut.get(name)
if (!targetEdge || !targetEdge.peer) {
return target
}
}
}
module.exports = deepestNestingTarget
@@ -0,0 +1,150 @@
// Do not rely on package._fields, so that we don't throw
// false failures if a tree is generated by other clients.
// Only relies on child.resolved, which MAY come from
// client-specific package.json meta _fields, but most of
// the time will be pulled out of a lockfile
const semver = require('semver')
const npa = require('npm-package-arg')
const { relative } = require('node:path')
const fromPath = require('./from-path.js')
const depValid = (child, requested, requestor) => {
// NB: we don't do much to verify 'tag' type requests.
// Just verify that we got a remote resolution. Presumably, it
// came from a registry and was tagged at some point.
if (typeof requested === 'string') {
try {
// tarball/dir must have resolved to the same tgz on disk, but for
// file: deps that depend on other files/dirs, we must resolve the
// location based on the *requestor* file/dir, not where it ends up.
// '' is equivalent to '*'
requested = npa.resolve(child.name, requested || '*', fromPath(requestor, requestor.edgesOut.get(child.name)))
} catch (er) {
// Not invalid because the child doesn't match, but because
// the spec itself is not supported. Nothing would match,
// so the edge is definitely not valid and never can be.
er.dependency = child.name
er.requested = requested
requestor.errors.push(er)
return false
}
}
// if the lockfile is super old, or hand-modified,
// then it's possible to hit this state.
if (!requested) {
const er = new Error('Invalid dependency specifier')
er.dependency = child.name
er.requested = requested
requestor.errors.push(er)
return false
}
switch (requested.type) {
case 'range':
if (requested.fetchSpec === '*') {
return true
}
// fallthrough
case 'version':
// if it's a version or a range other than '*', semver it
return semver.satisfies(child.version, requested.fetchSpec, true)
case 'directory':
return linkValid(child, requested, requestor)
case 'file':
return tarballValid(child, requested, requestor)
case 'alias':
// check that the alias target is valid
return depValid(child, requested.subSpec, requestor)
case 'tag':
// if it's a tag, we just verify that it has a tarball resolution
// presumably, it came from the registry and was tagged at some point
return child.resolved && npa(child.resolved).type === 'remote'
case 'remote':
// verify that we got it from the desired location
return child.resolved === requested.fetchSpec
case 'git': {
// if it's a git type, verify that they're the same repo
//
// if it specifies a definite commit, then it must have the
// same commit to be considered the same repo
//
// if it has a #semver:<range> specifier, verify that the
// version in the package is in the semver range
const resRepo = npa(child.resolved || '')
const resHost = resRepo.hosted
const reqHost = requested.hosted
const reqCommit = /^[a-fA-F0-9]{40,64}$/.test(requested.gitCommittish || '')
const nc = { noCommittish: !reqCommit }
if (!resHost) {
if (resRepo.fetchSpec !== requested.fetchSpec) {
return false
}
} else {
if (reqHost?.ssh(nc) !== resHost.ssh(nc)) {
return false
}
}
if (!requested.gitRange) {
return true
}
return semver.satisfies(child.package.version, requested.gitRange, {
loose: true,
})
}
default: // impossible, just being cautious
break
}
const er = new Error('Unsupported dependency type')
er.dependency = child.name
er.requested = requested
requestor.errors.push(er)
return false
}
const linkValid = (child, requested, requestor) => {
const isLink = !!child.isLink
// if we're installing links and the node is a link, then it's invalid because we want
// a real node to be there. Except for workspaces. They are always links.
if (requestor.installLinks && !child.isWorkspace) {
return !isLink
}
// directory must be a link to the specified folder
return isLink && relative(child.realpath, requested.fetchSpec) === ''
}
const tarballValid = (child, requested) => {
if (child.isLink) {
return false
}
if (child.resolved) {
return child.resolved.replace(/\\/g, '/') === `file:${requested.fetchSpec.replace(/\\/g, '/')}`
}
// if we have a legacy mutated package.json file. we can't be 100%
// sure that it resolved to the same file, but if it was the same
// request, that's a pretty good indicator of sameness.
if (child.package._requested) {
return child.package._requested.saveSpec === requested.saveSpec
}
// ok, we're probably dealing with some legacy cruft here, not much
// we can do at this point unfortunately.
return false
}
module.exports = (child, requested, accept, requestor) =>
depValid(child, requested, requestor) ||
(typeof accept === 'string' ? depValid(child, accept, requestor) : false)
@@ -0,0 +1,330 @@
// a tree representing the difference between two trees
// A Diff node's parent is not necessarily the parent of
// the node location it refers to, but rather the highest level
// node that needs to be either changed or removed.
// Thus, the root Diff node is the shallowest change required
// for a given branch of the tree being mutated.
const { depth } = require('treeverse')
const { existsSync } = require('node:fs')
const ssri = require('ssri')
class Diff {
constructor ({ actual, ideal, filterSet, shrinkwrapInflated, omit }) {
this.omit = omit
this.filterSet = filterSet
this.shrinkwrapInflated = shrinkwrapInflated
this.children = []
this.actual = actual
this.ideal = ideal
if (this.ideal) {
this.resolved = this.ideal.resolved
this.integrity = this.ideal.integrity
}
this.action = getAction(this)
this.parent = null
// the set of leaf nodes that we rake up to the top level
this.leaves = []
// the set of nodes that don't change in this branch of the tree
this.unchanged = []
// the set of nodes that will be removed in this branch of the tree
this.removed = []
}
static calculate ({
actual,
ideal,
filterNodes = [],
shrinkwrapInflated = new Set(),
omit = new Set(),
}) {
// if there's a filterNode, then:
// - get the path from the root to the filterNode. The root or
// root.target should have an edge either to the filterNode or
// a link to the filterNode. If not, abort. Add the path to the
// filterSet.
// - Add set of Nodes depended on by the filterNode to filterSet.
// - Anything outside of that set should be ignored by getChildren
const filterSet = new Set()
const extraneous = new Set()
for (const filterNode of filterNodes) {
const { root } = filterNode
if (root !== ideal && root !== actual) {
throw new Error('invalid filterNode: outside idealTree/actualTree')
}
const rootTarget = root.target
const edge = [...rootTarget.edgesOut.values()].filter(e => {
return e.to && (e.to === filterNode || e.to.target === filterNode)
})[0]
filterSet.add(root)
filterSet.add(rootTarget)
filterSet.add(ideal)
filterSet.add(actual)
if (edge && edge.to) {
filterSet.add(edge.to)
filterSet.add(edge.to.target)
}
filterSet.add(filterNode)
depth({
tree: filterNode,
visit: node => filterSet.add(node),
getChildren: node => {
const orig = node
node = node.target
const loc = node.location
const idealNode = ideal.inventory.get(loc)
const ideals = !idealNode ? []
: [...idealNode.edgesOut.values()].filter(e => e.to).map(e => e.to)
const actualNode = actual.inventory.get(loc)
const actuals = !actualNode ? []
: [...actualNode.edgesOut.values()].filter(e => e.to).map(e => e.to)
if (actualNode) {
for (const child of actualNode.children.values()) {
if (child.extraneous) {
extraneous.add(child)
}
}
}
const result = ideals.concat(actuals)
// Include link targets so store entries end up in filterSet
if (orig.isLink) {
result.push(node)
}
return result
},
})
}
for (const extra of extraneous) {
filterSet.add(extra)
}
return depth({
tree: new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit }),
getChildren,
leave,
})
}
}
const getAction = ({ actual, ideal }) => {
if (!ideal) {
return 'REMOVE'
}
// bundled meta-deps are copied over to the ideal tree when we visit it,
// so they'll appear to be missing here. There's no need to handle them
// in the diff, though, because they'll be replaced at reify time anyway
// Otherwise, add the missing node.
if (!actual) {
return ideal.inDepBundle ? null : 'ADD'
}
// always ignore the root node
if (ideal.isRoot && actual.isRoot) {
return null
}
// if the versions don't match, it's a change no matter what
if (ideal.version !== actual.version) {
return 'CHANGE'
}
const binsExist = ideal.binPaths.every((path) => existsSync(path))
// top nodes, links, and git deps won't have integrity, but do have resolved
// if neither node has integrity, the bins exist, and either (a) neither
// node has a resolved value or (b) they both do and match, then we can
// leave this one alone since we already know the versions match due to
// the condition above. The "neither has resolved" case (a) cannot be
// treated as a 'mark CHANGE and refetch', because shrinkwraps, bundles,
// and link deps may lack this information, and we don't want to try to
// go to the registry for something that isn't there.
const noIntegrity = !ideal.integrity && !actual.integrity
const noResolved = !ideal.resolved && !actual.resolved
const resolvedMatch = ideal.resolved && ideal.resolved === actual.resolved
if (noIntegrity && binsExist && (resolvedMatch || noResolved)) {
return null
}
// otherwise, verify that it's the same bits
// note that if ideal has integrity, and resolved doesn't, we treat
// that as a 'change', so that it gets re-fetched and locked down.
const integrityMismatch = !ideal.integrity || !actual.integrity ||
!ssri.parse(ideal.integrity).match(actual.integrity)
if (integrityMismatch || !binsExist) {
return 'CHANGE'
}
return null
}
const allChildren = node => {
if (!node) {
return new Map()
}
// if the node is root, and also a link, then what we really
// want is to traverse the target's children
if (node.isRoot && node.isLink) {
return allChildren(node.target)
}
const kids = new Map()
for (const n of [node, ...node.fsChildren]) {
for (const kid of n.children.values()) {
kids.set(kid.path, kid)
}
}
return kids
}
// functions for the walk options when we traverse the trees
// to create the diff tree
const getChildren = diff => {
const children = []
const {
actual,
ideal,
unchanged,
removed,
filterSet,
shrinkwrapInflated,
omit,
} = diff
// Note: we DON'T diff fsChildren themselves, because they are either
// included in the package contents, or part of some other project, and
// will never appear in legacy shrinkwraps anyway. but we _do_ include the
// child nodes of fsChildren, because those are nodes that we are typically
// responsible for installing.
const actualKids = allChildren(actual)
const idealKids = allChildren(ideal)
if (ideal && ideal.hasShrinkwrap && !shrinkwrapInflated.has(ideal)) {
// Guaranteed to get a diff.leaves here, because we always
// be called with a proper Diff object when ideal has a shrinkwrap
// that has not been inflated.
diff.leaves.push(diff)
return children
}
const paths = new Set([...actualKids.keys(), ...idealKids.keys()])
for (const path of paths) {
const actual = actualKids.get(path)
const ideal = idealKids.get(path)
diffNode({
actual,
ideal,
children,
unchanged,
removed,
filterSet,
shrinkwrapInflated,
omit,
})
}
if (diff.leaves && !children.length) {
diff.leaves.push(diff)
}
return children
}
const diffNode = ({
actual,
ideal,
children,
unchanged,
removed,
filterSet,
shrinkwrapInflated,
omit,
}) => {
if (filterSet.size && !(filterSet.has(ideal) || filterSet.has(actual))) {
return
}
if (ideal?.shouldOmit?.(omit)) {
ideal.inert = true
}
// Treat inert nodes as undefined for the purposes of diffing.
if (ideal?.inert) {
ideal = undefined
}
if (!actual && !ideal) {
return
}
const action = getAction({ actual, ideal })
// if it's a match, then get its children
// otherwise, this is the child diff node
if (action || (!shrinkwrapInflated.has(ideal) && ideal.hasShrinkwrap)) {
if (action === 'REMOVE') {
removed.push(actual)
}
children.push(new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit }))
} else {
unchanged.push(ideal)
// !*! Weird dirty hack warning !*!
//
// Bundled deps aren't loaded in the ideal tree, because we don't know
// what they are going to be without unpacking. Swap them over now if
// the bundling node isn't changing, so we don't prune them later.
//
// It's a little bit dirty to be doing this here, since it means that
// diffing trees can mutate them, but otherwise we have to walk over
// all unchanging bundlers and correct the diff later, so it's more
// efficient to just fix it while we're passing through already.
//
// Note that moving over a bundled dep will break the links to other
// deps under this parent, which may have been transitively bundled.
// Breaking those links means that we'll no longer see the transitive
// dependency, meaning that it won't appear as bundled any longer!
// In order to not end up dropping transitively bundled deps, we have
// to get the list of nodes to move, then move them all at once, rather
// than moving them one at a time in the first loop.
const bd = ideal.package.bundleDependencies
if (actual && bd && bd.length) {
const bundledChildren = []
for (const node of actual.children.values()) {
if (node.inBundle) {
bundledChildren.push(node)
}
}
for (const node of bundledChildren) {
node.parent = ideal
}
}
children.push(...getChildren({
actual,
ideal,
unchanged,
removed,
filterSet,
shrinkwrapInflated,
omit,
}))
}
}
// set the parentage in the leave step so that we aren't attaching
// child nodes only to remove them later. also bubble up the unchanged
// nodes so that we can move them out of staging in the reification step.
const leave = (diff, children) => {
children.forEach(kid => {
kid.parent = diff
diff.leaves.push(...kid.leaves)
diff.unchanged.push(...kid.unchanged)
diff.removed.push(...kid.removed)
})
diff.children = children
return diff
}
module.exports = Diff
@@ -0,0 +1,364 @@
// An edge in the dependency graph
// Represents a dependency relationship of some kind
const util = require('node:util')
const npa = require('npm-package-arg')
const depValid = require('./dep-valid.js')
const OverrideSet = require('./override-set.js')
class ArboristEdge {
constructor (edge) {
this.name = edge.name
this.spec = edge.spec
this.type = edge.type
const edgeFrom = edge.from?.location
const edgeTo = edge.to?.location
const override = edge.overrides?.value
if (edgeFrom != null) {
this.from = edgeFrom
}
if (edgeTo) {
this.to = edgeTo
}
if (edge.error) {
this.error = edge.error
}
if (edge.peerConflicted) {
this.peerConflicted = true
}
if (override) {
this.overridden = override
}
}
}
class Edge {
#accept
#error
#explanation
#from
#name
#spec
#to
#type
static types = Object.freeze([
'prod',
'dev',
'optional',
'peer',
'peerOptional',
'workspace',
])
// XXX where is this used?
static errors = Object.freeze([
'DETACHED',
'MISSING',
'PEER LOCAL',
'INVALID',
])
constructor (options) {
const { type, name, spec, accept, from, overrides } = options
// XXX are all of these error states even possible?
if (typeof spec !== 'string') {
throw new TypeError('must provide string spec')
}
if (!Edge.types.includes(type)) {
throw new TypeError(`invalid type: ${type}\n(valid types are: ${Edge.types.join(', ')})`)
}
if (type === 'workspace' && npa(spec).type !== 'directory') {
throw new TypeError('workspace edges must be a symlink')
}
if (typeof name !== 'string') {
throw new TypeError('must provide dependency name')
}
if (!from) {
throw new TypeError('must provide "from" node')
}
if (accept !== undefined) {
if (typeof accept !== 'string') {
throw new TypeError('accept field must be a string if provided')
}
this.#accept = accept || '*'
}
if (overrides !== undefined) {
this.overrides = overrides
}
this.#name = name
this.#type = type
this.#spec = spec
this.#explanation = null
this.#from = from
from.edgesOut.get(this.#name)?.detach()
from.addEdgeOut(this)
this.reload(true)
this.peerConflicted = false
}
satisfiedBy (node) {
if (node.name !== this.#name || !this.#from) {
return false
}
// NOTE: this condition means we explicitly do not support overriding
// bundled or shrinkwrapped dependencies
if (node.hasShrinkwrap || node.inShrinkwrap || node.inBundle) {
return depValid(node, this.rawSpec, this.#accept, this.#from)
}
// If there's no override we just use the spec.
if (!this.overrides?.keySpec) {
return depValid(node, this.spec, this.#accept, this.#from)
}
// There's some override. If the target node satisfies the overriding spec
// then it's okay.
if (depValid(node, this.spec, this.#accept, this.#from)) {
return true
}
// If it doesn't, then it should at least satisfy the original spec.
if (!depValid(node, this.rawSpec, this.#accept, this.#from)) {
return false
}
// It satisfies the original spec, not the overriding spec. We need to make
// sure it doesn't use the overridden spec.
// For example:
// we might have an ^8.0.0 rawSpec, and an override that makes
// keySpec=8.23.0 and the override value spec=9.0.0.
// If the node is 9.0.0, then it's okay because it's consistent with spec.
// If the node is 8.24.0, then it's okay because it's consistent with the rawSpec.
// If the node is 8.23.0, then it's not okay because even though it's consistent
// with the rawSpec, it's also consistent with the keySpec.
// So we're looking for ^8.0.0 or 9.0.0 and not 8.23.0.
return !depValid(node, this.overrides.keySpec, this.#accept, this.#from)
}
// return the edge data, and an explanation of how that edge came to be here
explain (seen = []) {
if (!this.#explanation) {
const explanation = {
type: this.#type,
name: this.#name,
spec: this.spec,
}
if (this.rawSpec !== this.spec) {
explanation.rawSpec = this.rawSpec
explanation.overridden = true
}
if (this.bundled) {
explanation.bundled = this.bundled
}
if (this.error) {
explanation.error = this.error
}
if (this.#from) {
explanation.from = this.#from.explain(null, seen)
}
this.#explanation = explanation
}
return this.#explanation
}
get bundled () {
return !!this.#from?.package?.bundleDependencies?.includes(this.#name)
}
get workspace () {
return this.#type === 'workspace'
}
get prod () {
return this.#type === 'prod'
}
get dev () {
return this.#type === 'dev'
}
get optional () {
return this.#type === 'optional' || this.#type === 'peerOptional'
}
get peer () {
return this.#type === 'peer' || this.#type === 'peerOptional'
}
get type () {
return this.#type
}
get name () {
return this.#name
}
get rawSpec () {
return this.#spec
}
get spec () {
if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.#name) {
if (this.overrides.value.startsWith('$')) {
const ref = this.overrides.value.slice(1)
let pkg = this.#from?.sourceReference
? this.#from?.sourceReference.root.package
: this.#from?.root?.package
let specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
// If the package isn't found in the root package, fall back to the local package.
if (!specValue) {
pkg = this.#from?.package
specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
}
if (specValue) {
return specValue
}
throw new Error(`Unable to resolve reference ${this.overrides.value}`)
}
return this.overrides.value
}
return this.#spec
}
#calculateReferentialOverrideSpec (ref, pkg) {
if (pkg.devDependencies?.[ref]) {
return pkg.devDependencies[ref]
}
if (pkg.optionalDependencies?.[ref]) {
return pkg.optionalDependencies[ref]
}
if (pkg.dependencies?.[ref]) {
return pkg.dependencies[ref]
}
if (pkg.peerDependencies?.[ref]) {
return pkg.peerDependencies[ref]
}
}
get accept () {
return this.#accept
}
get valid () {
return !this.error
}
get missing () {
return this.error === 'MISSING'
}
get invalid () {
return this.error === 'INVALID'
}
get peerLocal () {
return this.error === 'PEER LOCAL'
}
get error () {
if (!this.#error) {
if (!this.#to) {
if (this.optional) {
this.#error = null
} else {
this.#error = 'MISSING'
}
} else if (this.peer && this.#from === this.#to.parent && !this.#from?.isTop) {
this.#error = 'PEER LOCAL'
} else if (!this.satisfiedBy(this.#to)) {
this.#error = 'INVALID'
} else if (this.overrides && this.#to.overrides && this.#to.edgesOut.size && OverrideSet.doOverrideSetsConflict(this.overrides, this.#to.overrides)) {
// Check for conflicts between the edge's override set and the target node's override set.
// This catches cases where different parts of the tree have genuinely incompatible
// version requirements for the same package.
// The improved conflict detection uses semantic comparison (checking for incompatible
// version ranges) rather than pure structural equality, avoiding false positives from:
// - Reference overrides ($syntax) that resolve to compatible versions
// - Peer dependencies with different but compatible override contexts
// Note: We only check if the target has dependencies (edgesOut.size > 0), since
// override conflicts are only relevant if the target has its own dependencies.
this.#error = 'INVALID'
} else {
this.#error = 'OK'
}
}
if (this.#error === 'OK') {
return null
}
return this.#error
}
reload (hard = false) {
this.#explanation = null
let needToUpdateOverrideSet = false
let newOverrideSet
let oldOverrideSet
if (this.#from?.overrides) {
newOverrideSet = this.#from.overrides.getEdgeRule(this)
if (newOverrideSet && !newOverrideSet.isEqual(this.overrides)) {
// If there's a new different override set we need to propagate it to the nodes.
// If we're deleting the override set then there's no point propagating it right now since it will be filled with another value later.
needToUpdateOverrideSet = true
oldOverrideSet = this.overrides
this.overrides = newOverrideSet
}
} else {
delete this.overrides
}
const newTo = this.#from?.resolve(this.#name)
if (newTo !== this.#to) {
if (this.#to) {
this.#to.deleteEdgeIn(this)
}
this.#to = newTo
this.#error = null
if (this.#to) {
this.#to.addEdgeIn(this)
}
} else if (hard) {
this.#error = null
} else if (needToUpdateOverrideSet && this.#to) {
// Propagate the new override set to the target node.
this.#to.updateOverridesEdgeInRemoved(oldOverrideSet)
this.#to.updateOverridesEdgeInAdded(newOverrideSet)
}
}
detach () {
this.#explanation = null
if (this.#to) {
this.#to.deleteEdgeIn(this)
}
this.#from?.edgesOut.delete(this.#name)
this.#to = null
this.#error = 'DETACHED'
this.#from = null
}
get from () {
return this.#from
}
get to () {
return this.#to
}
toJSON () {
return new ArboristEdge(this)
}
[util.inspect.custom] () {
return this.toJSON()
}
}
module.exports = Edge
@@ -0,0 +1,30 @@
// file dependencies need their dependencies resolved based on the location
// where the tarball was found, not the location where they end up getting
// installed. directory (ie, symlink) deps also need to be resolved based on
// their targets, but that's what realpath is
const { dirname } = require('node:path')
const npa = require('npm-package-arg')
const fromPath = (node, edge) => {
if (edge && edge.overrides && edge.overrides.name === edge.name && edge.overrides.value) {
// fromPath could be called with a node that has a virtual root, if that
// happens we want to make sure we get the real root node when overrides
// are in use. this is to allow things like overriding a dependency with a
// tarball file that's a relative path from the project root
if (node.sourceReference) {
return node.sourceReference.root.realpath
}
return node.root.realpath
}
if (node.resolved) {
const spec = npa(node.resolved)
if (spec?.type === 'file') {
return dirname(spec.fetchSpec)
}
}
return node.realpath
}
module.exports = fromPath
@@ -0,0 +1,43 @@
// Given a set of nodes in a tree, and a filter function to test
// incoming edges to the dep set that should be ignored otherwise.
//
// find the set of deps that are only depended upon by nodes in the set, or
// their dependencies, or edges that are ignored.
//
// Used when figuring out what to prune when replacing a node with a newer
// version, or when an optional dep fails to install.
const gatherDepSet = (set, edgeFilter) => {
const deps = new Set(set)
// add the full set of dependencies. note that this loop will continue
// as the deps set increases in size.
for (const node of deps) {
for (const edge of node.edgesOut.values()) {
if (edge.to && edgeFilter(edge)) {
deps.add(edge.to)
}
}
}
// now remove all nodes in the set that have a dependent outside the set
// if any change is made, then re-check
// continue until no changes made, or deps set evaporates fully.
let changed = true
while (changed === true && deps.size > 0) {
changed = false
for (const dep of deps) {
for (const edge of dep.edgesIn) {
if (!deps.has(edge.from) && edgeFilter(edge)) {
changed = true
deps.delete(dep)
break
}
}
}
}
return deps
}
module.exports = gatherDepSet
@@ -0,0 +1,6 @@
module.exports = require('./arborist/index.js')
module.exports.Arborist = module.exports
module.exports.Node = require('./node.js')
module.exports.Link = require('./link.js')
module.exports.Edge = require('./edge.js')
module.exports.Shrinkwrap = require('./shrinkwrap.js')
@@ -0,0 +1,88 @@
const { isNodeGypPackage } = require('@npmcli/node-gyp')
// Returns the install-relevant lifecycle scripts that would run for a
// given arborist Node, or `{}` if there are none.
//
// Includes:
// - explicit preinstall/install/postinstall
// - prepare, but only for non-registry sources (git, file, link, remote)
// - synthetic `node-gyp rebuild`, when `binding.gyp` is present on disk
// and the package does not opt out via `gypfile: false` or define its
// own install / preinstall script
// Lifecycle-script enumeration boundary.
//
// IMPORTANT: this helper decides whether `prepare` should be included
// in the enumerated install scripts (true for non-registry sources only).
// It is NOT a policy-matching predicate. The policy matcher in
// script-allowed.js uses `isRegistryNode`, which is strictly tied to
// versionFromTgz(node.resolved). The two helpers exist separately on
// purpose:
//
// - `hasNonRegistryShape` (here): "should we consider running prepare
// on this node?" — a yes/no for what to enumerate.
// - `isRegistryNode` (script-allowed.js): "do we trust this node's
// identity enough to apply a policy entry?" — a security check.
//
// The looser fallback here (treating unknown-resolved nodes as registry,
// thus skipping `prepare`) is the safer default for enumeration: we'd
// rather omit a script we should have run than synthesise one for a
// non-registry source we couldn't confirm. The policy matcher's stricter
// behaviour is correct for its boundary; the two helpers must not be
// merged.
const hasNonRegistryShape = (node) => {
if (typeof node.isRegistryDependency === 'boolean') {
return !node.isRegistryDependency
}
if (!node.resolved) {
return false
}
return !/^https?:\/\/[^/]+\/.+\/-\/[^/]+-\d/.test(node.resolved)
}
const getInstallScripts = async (node) => {
/* istanbul ignore next: arborist Nodes always carry a `package` object;
defensive fallbacks for non-arborist callers. */
const pkg = node.package || {}
/* istanbul ignore next */
const scripts = pkg.scripts || {}
const collected = {}
if (scripts.preinstall) {
collected.preinstall = scripts.preinstall
}
if (scripts.install) {
collected.install = scripts.install
}
if (scripts.postinstall) {
collected.postinstall = scripts.postinstall
}
if (scripts.prepare && hasNonRegistryShape(node)) {
collected.prepare = scripts.prepare
}
const hasExplicitGypGate = !!(collected.preinstall || collected.install)
if (
!hasExplicitGypGate &&
pkg.gypfile !== false &&
await isNodeGypPackage(node.path).catch(() => false)
) {
collected.install = 'node-gyp rebuild'
}
// Lockfile-only nodes (e.g. `npm ci` before reify) carry
// `hasInstallScript: true` but no enumerated scripts: the lockfile
// records the presence flag but never the script bodies. Without this
// fallback the strict-allow-scripts preflight would miss them entirely
// and let postinstall run. We can't recover the real script body
// without fetching the manifest, so emit a sentinel describing that
// install scripts are present.
if (Object.keys(collected).length === 0 && node.hasInstallScript === true) {
collected.install = '(install scripts present)'
}
return collected
}
module.exports = getInstallScripts
module.exports.getInstallScripts = getInstallScripts
@@ -0,0 +1,137 @@
// a class to manage an inventory and set of indexes of a set of objects based on specific fields.
const { hasOwnProperty } = Object.prototype
const debug = require('./debug.js')
const keys = ['name', 'license', 'funding', 'realpath', 'packageName']
class Inventory extends Map {
#index
constructor () {
super()
this.#index = new Map()
for (const key of keys) {
this.#index.set(key, new Map())
}
}
// XXX where is this used?
get primaryKey () {
return 'location'
}
// XXX where is this used?
get indexes () {
return [...keys]
}
* filter (fn) {
for (const node of this.values()) {
if (fn(node)) {
yield node
}
}
}
add (node) {
const root = super.get('')
if (root && node.root !== root && node.root !== root.root) {
debug(() => {
throw Object.assign(new Error('adding external node to inventory'), {
root: root.path,
node: node.path,
nodeRoot: node.root.path,
})
})
return
}
const current = super.get(node.location)
if (current) {
if (current === node) {
return
}
this.delete(current)
}
super.set(node.location, node)
for (const [key, map] of this.#index.entries()) {
let val
if (hasOwnProperty.call(node, key)) {
// if the node has the value, use it even if it's false
val = node[key]
} else if (key === 'license' && node.package) {
// handling for the outdated "licenses" array, just pick the first one
// also support the alternative spelling "licence"
if (node.package.license) {
val = node.package.license
} else if (node.package.licence) {
val = node.package.licence
} else if (Array.isArray(node.package.licenses)) {
val = node.package.licenses[0]
} else if (Array.isArray(node.package.licences)) {
val = node.package.licences[0]
}
} else if (node[key]) {
val = node[key]
} else {
val = node.package?.[key]
}
if (val && typeof val === 'object') {
// We currently only use license and funding
/* istanbul ignore next - not used */
if (key === 'license') {
val = val.type
} else if (key === 'funding') {
val = val.url
}
}
if (!map.has(val)) {
map.set(val, new Set())
}
map.get(val).add(node)
}
}
delete (node) {
if (!this.has(node)) {
return
}
super.delete(node.location)
for (const [key, map] of this.#index.entries()) {
let val
if (node[key] !== undefined) {
val = node[key]
} else {
val = node.package?.[key]
}
const set = map.get(val)
if (set) {
set.delete(node)
if (set.size === 0) {
map.delete(node[key])
}
}
}
}
query (key, val) {
const map = this.#index.get(key)
if (arguments.length === 2) {
if (map.has(val)) {
return map.get(val)
}
return new Set()
}
return map.keys()
}
has (node) {
return super.get(node.location) === node
}
set () {
throw new Error('direct set() not supported, use inventory.add(node)')
}
}
module.exports = Inventory
@@ -0,0 +1,138 @@
// Alternate versions of different classes that we use for isolated mode
const CaseInsensitiveMap = require('./case-insensitive-map.js')
const { resolve } = require('node:path')
// fake lib/inventory.js
class IsolatedInventory extends Map {
query () {
return []
}
}
// fake lib/node.js
class IsolatedNode {
binPaths = []
children = new CaseInsensitiveMap()
edgesIn = new Set()
edgesOut = new CaseInsensitiveMap()
fsChildren = new Set()
hasShrinkwrap = false
integrity = null
inventory = new IsolatedInventory()
isInStore = false
linksIn = new Set()
meta = { loadedFromDisk: false }
optional = false
parent = null
root = null
tops = new Set()
workspaces = new Map()
constructor (options) {
this.location = options.location
this.name = options.name
this.package = options.package
this.path = options.path
this.realpath = !this.isLink ? this.path : resolve(options.realpath)
if (options.parent) {
this.parent = options.parent
}
if (options.resolved) {
this.resolved = options.resolved
}
if (options.root) {
this.root = options.root
}
if (options.isInStore) {
this.isInStore = true
}
if (options.optional) {
this.optional = true
}
}
get isRoot () {
return this === this.root
}
// The idealGraph is where this is set to true
get isProjectRoot () {
return false
}
get inDepBundle () {
return false
}
get isLink () {
return false
}
get isTop () {
return !this.parent
}
/* istanbul ignore next -- emulate lib/node.js */
get global () {
return false
}
get globalTop () {
return false
}
/* istanbul ignore next -- emulate lib/node.js */
set target (t) {
// nop
// In the real lib/node.js this throws in debug mode
}
get target () {
return this
}
/* istanbul ignore next -- emulate lib/node.js */
getBundler () {
return null
}
/* istanbul ignore next -- emulate lib/node.js */
get hasInstallScript () {
const { hasInstallScript, scripts } = this.package
const { install, preinstall, postinstall } = scripts || {}
return !!(hasInstallScript || install || preinstall || postinstall)
}
get version () {
return this.package.version
}
}
// fake lib/link.js
class IsolatedLink extends IsolatedNode {
#target
isStoreLink = false
constructor (options) {
super(options)
this.#target = options.target
if (options.isStoreLink) {
this.isStoreLink = true
}
}
get isLink () {
return true
}
set target (t) {
this.#target = t
}
get target () {
return this.#target
}
}
module.exports = { IsolatedNode, IsolatedLink }
@@ -0,0 +1,146 @@
const relpath = require('./relpath.js')
const Node = require('./node.js')
const _loadDeps = Symbol.for('Arborist.Node._loadDeps')
const _target = Symbol.for('_target')
const { dirname } = require('node:path')
// defined by Node class
const _delistFromMeta = Symbol.for('_delistFromMeta')
const _refreshLocation = Symbol.for('_refreshLocation')
class Link extends Node {
constructor (options) {
const { root, realpath, target, parent, fsParent, isStoreLink } = options
if (!realpath && !(target && target.path)) {
throw new TypeError('must provide realpath for Link node')
}
super({
...options,
realpath: realpath || target.path,
root: root || (parent ? parent.root
: fsParent ? fsParent.root
: target ? target.root
: null),
})
this.isStoreLink = isStoreLink || false
if (target) {
this.target = target
} else if (this.realpath === this.root.path) {
this.target = this.root
} else {
this.target = new Node({
...options,
path: realpath,
parent: null,
fsParent: null,
root: this.root,
})
}
}
get version () {
return this.target ? this.target.version : this.package.version || ''
}
get target () {
return this[_target]
}
set target (target) {
const current = this[_target]
if (target === current) {
return
}
if (!target) {
if (current && current.linksIn) {
current.linksIn.delete(this)
}
if (this.path) {
this[_delistFromMeta]()
this[_target] = null
this.package = {}
this[_refreshLocation]()
} else {
this[_target] = null
}
return
}
if (!this.path) {
// temp node pending assignment to a tree
// we know it's not in the inventory yet, because no path.
if (target.path) {
this.realpath = target.path
} else {
target.path = target.realpath = this.realpath
}
target.root = this.root
this[_target] = target
target.linksIn.add(this)
this.package = target.package
return
}
// have to refresh metadata, because either realpath or package
// is very likely changing.
this[_delistFromMeta]()
this.package = target.package
this.realpath = target.path
this[_refreshLocation]()
target.root = this.root
}
// a link always resolves to the relative path to its target
get resolved () {
// the path/realpath guard is there for the benefit of setting
// these things in the "wrong" order
return this.path && this.realpath
? `file:${relpath(dirname(this.path), this.realpath)}`
: null
}
set resolved (r) {}
// deps are resolved on the target, not the Link
// so this is a no-op
[_loadDeps] () {}
// When a Link receives overrides (via edgesIn), forward them to the target node which holds the actual edgesOut — but only when the OverrideSet has at least one rule that names a dep the target actually depends on.
// Without this scope, the link forwards a generic ancestor OverrideSet that has no real effect on the target's edges, but still flips the target to "has overrides", which changes downstream `canReplaceWith` / placement decisions and causes `npm ci` to re-resolve lockfile-pinned edges from the registry.
// See npm/cli#9357.
recalculateOutEdgesOverrides () {
if (!this.target || !this.overrides) {
return
}
let hasMatchingRule = false
for (const rule of this.overrides.ruleset.values()) {
if (this.target.edgesOut.has(rule.name)) {
hasMatchingRule = true
break
}
}
if (!hasMatchingRule) {
return
}
this.target.updateOverridesEdgeInAdded(this.overrides)
}
// links can't have children, only their targets can
// fix it to an empty list so that we can still call
// things that iterate over them, just as a no-op
get children () {
return new Map()
}
set children (c) {}
get isLink () {
return true
}
}
module.exports = Link
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,32 @@
// when an optional dep fails to install, we need to remove the branch of the
// graph up to the first optionalDependencies, as well as any nodes that are
// only required by other nodes in the set.
//
// This function finds the set of nodes that will need to be removed in that
// case.
//
// Note that this is *only* going to work with trees where calcDepFlags
// has been called, because we rely on the node.optional flag.
const gatherDepSet = require('./gather-dep-set.js')
const optionalSet = node => {
// start with the node, then walk up the dependency graph until we
// get to the boundaries that define the optional set. since the
// node is optional, we know that all paths INTO this area of the
// graph are optional, but there may be non-optional dependencies
// WITHIN the area.
const set = new Set([node])
for (const node of set) {
for (const edge of node.edgesIn) {
if (!edge.optional) {
set.add(edge.from)
}
}
}
// now that we've hit the boundary, gather the rest of the nodes in
// the optional section that don't have dependents outside the set.
return gatherDepSet(set, edge => !set.has(edge.to) && !edge.from?.inert)
}
module.exports = optionalSet
@@ -0,0 +1,11 @@
function overrideResolves (resolved, opts) {
const { omitLockfileRegistryResolved = false } = opts
if (omitLockfileRegistryResolved) {
return undefined
}
return resolved
}
module.exports = { overrideResolves }
@@ -0,0 +1,304 @@
const npa = require('npm-package-arg')
const semver = require('semver')
const { log } = require('proc-log')
class OverrideSet {
constructor ({ overrides, key, parent }) {
this.parent = parent
this.children = new Map()
if (typeof overrides === 'string') {
overrides = { '.': overrides }
}
// change a literal empty string to * so we can use truthiness checks on
// the value property later
if (overrides['.'] === '') {
overrides['.'] = '*'
}
if (parent) {
const spec = npa(key)
if (!spec.name) {
throw new Error(`Override without name: ${key}`)
}
this.name = spec.name
spec.name = ''
this.key = key
this.keySpec = spec.toString()
this.value = overrides['.'] || this.keySpec
}
for (const [key, childOverrides] of Object.entries(overrides)) {
if (key === '.') {
continue
}
const child = new OverrideSet({
parent: this,
key,
overrides: childOverrides,
})
this.children.set(child.key, child)
}
}
childrenAreEqual (other) {
if (this.children.size !== other.children.size) {
return false
}
for (const [key] of this.children) {
if (!other.children.has(key)) {
return false
}
if (this.children.get(key).value !== other.children.get(key).value) {
return false
}
if (!this.children.get(key).childrenAreEqual(other.children.get(key))) {
return false
}
}
return true
}
isEqual (other) {
if (this === other) {
return true
}
if (!other) {
return false
}
if (this.key !== other.key || this.value !== other.value) {
return false
}
if (!this.childrenAreEqual(other)) {
return false
}
if (!this.parent) {
return !other.parent
}
return this.parent.isEqual(other.parent)
}
getEdgeRule (edge) {
for (const rule of this.ruleset.values()) {
if (rule.name !== edge.name) {
continue
}
// if keySpec is * we found our override
if (rule.keySpec === '*') {
return rule
}
// We need to use the rawSpec here, because the spec has the overrides applied to it already.
// rawSpec can be undefined, so we need to use the fallback value of spec if it is.
let spec = npa(`${edge.name}@${edge.rawSpec || edge.spec}`)
if (spec.type === 'alias') {
spec = spec.subSpec
}
if (spec.type === 'git') {
if (spec.gitRange && semver.intersects(spec.gitRange, rule.keySpec)) {
return rule
}
continue
}
if (spec.type === 'range' || spec.type === 'version') {
if (semver.intersects(spec.fetchSpec, rule.keySpec)) {
return rule
}
continue
}
// if we got this far, the spec type is one of tag, directory or file
// which means we have no real way to make version comparisons, so we
// just accept the override
return rule
}
return this
}
getNodeRule (node) {
for (const rule of this.ruleset.values()) {
if (rule.name !== node.name) {
continue
}
if (semver.satisfies(node.version, rule.keySpec) ||
semver.satisfies(node.version, rule.value)) {
return rule
}
}
return this
}
getMatchingRule (node) {
for (const rule of this.ruleset.values()) {
if (rule.name !== node.name) {
continue
}
if (semver.satisfies(node.version, rule.keySpec) ||
semver.satisfies(node.version, rule.value)) {
return rule
}
}
return null
}
* ancestry () {
for (let ancestor = this; ancestor; ancestor = ancestor.parent) {
yield ancestor
}
}
get isRoot () {
return !this.parent
}
get ruleset () {
const ruleset = new Map()
for (const override of this.ancestry()) {
for (const kid of override.children.values()) {
if (!ruleset.has(kid.key)) {
ruleset.set(kid.key, kid)
}
}
if (!override.isRoot && !ruleset.has(override.key)) {
ruleset.set(override.key, override)
}
}
return ruleset
}
static findSpecificOverrideSet (first, second) {
for (let overrideSet = second; overrideSet; overrideSet = overrideSet.parent) {
if (overrideSet.isEqual(first)) {
return second
}
}
for (let overrideSet = first; overrideSet; overrideSet = overrideSet.parent) {
if (overrideSet.isEqual(second)) {
return first
}
}
// The override sets are incomparable (e.g. siblings like the "react" and "react-dom" children of the root override set). Check if they have semantically conflicting rules before treating this as an error.
if (this.haveConflictingRules(first, second)) {
log.silly('Conflicting override sets', first, second)
return undefined
}
// The override sets are structurally incomparable but have compatible rules. Fall back to their nearest common ancestor so the node still has a valid override set.
return this.findCommonAncestor(first, second)
}
static findCommonAncestor (first, second) {
const firstAncestors = []
for (const ancestor of first.ancestry()) {
firstAncestors.push(ancestor)
}
for (const secondAnc of second.ancestry()) {
for (const firstAnc of firstAncestors) {
if (firstAnc.isEqual(secondAnc)) {
return firstAnc
}
}
}
return null
}
static doOverrideSetsConflict (first, second) {
// If override sets contain one another then we can try to use the more specific one.
// If neither one is more specific, check for semantic conflicts.
const specificSet = this.findSpecificOverrideSet(first, second)
if (specificSet !== undefined) {
// One contains the other, so no conflict
return false
}
// The override sets are structurally incomparable, but this doesn't necessarily
// mean they conflict. We need to check if they have conflicting version requirements
// for any package that appears in both rulesets.
return this.haveConflictingRules(first, second)
}
static haveConflictingRules (first, second) {
// Get all rules from both override sets
const firstRules = first.ruleset
const secondRules = second.ruleset
// Check each package that appears in both rulesets
for (const [key, firstRule] of firstRules) {
const secondRule = secondRules.get(key)
if (!secondRule) {
// Package only appears in one ruleset, no conflict
continue
}
// Same rule object means no conflict
if (firstRule === secondRule || firstRule.isEqual(secondRule)) {
continue
}
// Both rulesets have rules for this package with different values.
// Check if the version requirements are actually incompatible.
const firstValue = firstRule.value
const secondValue = secondRule.value
// If either value is a reference (starts with $), we can't determine
// compatibility here - the reference might resolve to compatible versions.
// We defer to runtime resolution rather than failing early.
if (firstValue.startsWith('$') || secondValue.startsWith('$')) {
continue
}
// Check if the version ranges are compatible using semver
// If both specify version ranges, they conflict only if they have no overlap
try {
const firstSpec = npa(`${firstRule.name}@${firstValue}`)
const secondSpec = npa(`${secondRule.name}@${secondValue}`)
// For range/version types, check if they intersect
if ((firstSpec.type === 'range' || firstSpec.type === 'version') &&
(secondSpec.type === 'range' || secondSpec.type === 'version')) {
// Check if the ranges intersect
const firstRange = firstSpec.fetchSpec
const secondRange = secondSpec.fetchSpec
// If the ranges don't intersect, we have a real conflict
if (!semver.intersects(firstRange, secondRange)) {
log.silly('Found conflicting override rules', {
package: firstRule.name,
first: firstValue,
second: secondValue,
})
return true
}
}
// For other types (git, file, directory, tag), we can't easily determine
// compatibility, so we conservatively assume no conflict
} catch {
// If we can't parse the specs, conservatively assume no conflict
// Real conflicts will be caught during dependency resolution
}
}
// No conflicting rules found
return false
}
}
module.exports = OverrideSet
@@ -0,0 +1,77 @@
const { LRUCache } = require('lru-cache')
const { getHeapStatistics } = require('node:v8')
const { log } = require('proc-log')
// This is an in-memory cache that Pacote uses for packuments.
// Packuments are usually cached on disk. This allows for rapid re-requests
// of the same packument to bypass disk reads. The tradeoff here is memory
// usage for disk reads.
class PackumentCache extends LRUCache {
static #heapLimit = Math.floor(getHeapStatistics().heap_size_limit)
#sizeKey
#disposed = new Set()
#log (...args) {
log.silly('packumentCache', ...args)
}
constructor ({
// How much of this.#heapLimit to take up
heapFactor = 0.25,
// How much of this.#maxSize we allow any one packument to take up
// Anything over this is not cached
maxEntryFactor = 0.5,
sizeKey = '_contentLength',
} = {}) {
const maxSize = Math.floor(PackumentCache.#heapLimit * heapFactor)
const maxEntrySize = Math.floor(maxSize * maxEntryFactor)
super({
maxSize,
maxEntrySize,
sizeCalculation: (p) => {
// Don't cache if we don't know the size
// Some versions of pacote set this to `0`, newer versions set it to `null`
if (!p[sizeKey]) {
return maxEntrySize + 1
}
if (p[sizeKey] < 10_000) {
return p[sizeKey] * 2
}
if (p[sizeKey] < 1_000_000) {
return Math.floor(p[sizeKey] * 1.5)
}
// It is less beneficial to store a small amount of super large things
// at the cost of all other packuments.
return maxEntrySize + 1
},
dispose: (v, k) => {
this.#disposed.add(k)
this.#log(k, 'dispose')
},
})
this.#sizeKey = sizeKey
this.#log(`heap:${PackumentCache.#heapLimit} maxSize:${maxSize} maxEntrySize:${maxEntrySize}`)
}
set (k, v, ...args) {
// we use disposed only for a logging signal if we are setting packuments that
// have already been evicted from the cache previously. logging here could help
// us tune this in the future.
const disposed = this.#disposed.has(k)
/* istanbul ignore next - this doesnt happen consistently so hard to test without resorting to unit tests */
if (disposed) {
this.#disposed.delete(k)
}
this.#log(k, 'set', `size:${v[this.#sizeKey]} disposed:${disposed}`)
return super.set(k, v, ...args)
}
has (k, ...args) {
const has = super.has(k, ...args)
this.#log(k, `cache-${has ? 'hit' : 'miss'}`)
return has
}
}
module.exports = PackumentCache
@@ -0,0 +1,77 @@
// Given a node in a tree, return all of the peer dependency sets that
// it is a part of, with the entry (top or non-peer) edges into the sets
// identified.
//
// With this information, we can determine whether it is appropriate to
// replace the entire peer set with another (and remove the old one),
// push the set deeper into the tree, and so on.
//
// Returns a Map of { edge => Set(peerNodes) },
const peerEntrySets = node => {
// this is the union of all peer groups that the node is a part of
// later, we identify all of the entry edges, and create a set of
// 1 or more overlapping sets that this node is a part of.
const unionSet = new Set([node])
for (const node of unionSet) {
for (const edge of node.edgesOut.values()) {
if (edge.valid && edge.peer && edge.to) {
unionSet.add(edge.to)
}
}
for (const edge of node.edgesIn) {
if (edge.valid && edge.peer) {
unionSet.add(edge.from)
}
}
}
const entrySets = new Map()
for (const peer of unionSet) {
for (const edge of peer.edgesIn) {
// if not valid, it doesn't matter anyway. either it's been previously
// peerConflicted, or it's the thing we're interested in replacing.
if (!edge.valid) {
continue
}
// this is the entry point into the peer set
if (!edge.peer || edge.from.isTop) {
// get the subset of peer brought in by this peer entry edge
const sub = new Set([peer])
for (const peer of sub) {
for (const edge of peer.edgesOut.values()) {
if (edge.valid && edge.peer && edge.to) {
sub.add(edge.to)
}
}
}
// if this subset does not include the node we are focused on,
// then it is not relevant for our purposes. Example:
//
// a -> (b, c, d)
// b -> PEER(d) b -> d -> e -> f <-> g
// c -> PEER(f, h) c -> (f <-> g, h -> g)
// d -> PEER(e) d -> e -> f <-> g
// e -> PEER(f)
// f -> PEER(g)
// g -> PEER(f)
// h -> PEER(g)
//
// The unionSet(e) will include c, but we don't actually care about
// it. We only expanded to the edge of the peer nodes in order to
// find the entry edges that caused the inclusion of peer sets
// including (e), so we want:
// Map{
// Edge(a->b) => Set(b, d, e, f, g)
// Edge(a->d) => Set(d, e, f, g)
// }
if (sub.has(node)) {
entrySets.set(edge, sub)
}
}
}
}
return entrySets
}
module.exports = peerEntrySets
@@ -0,0 +1,569 @@
// Given a dep, a node that depends on it, and the edge representing that
// dependency, place the dep somewhere in the node's tree, and all of its
// peer dependencies.
//
// Handles all of the tree updating needed to place the dep, including
// removing replaced nodes, pruning now-extraneous or invalidated nodes,
// and saves a set of what was placed and what needs re-evaluation as
// a result.
const localeCompare = require('@isaacs/string-locale-compare')('en')
const { log } = require('proc-log')
const { redact } = require('@npmcli/redact')
const deepestNestingTarget = require('./deepest-nesting-target.js')
const CanPlaceDep = require('./can-place-dep.js')
const {
KEEP,
CONFLICT,
} = CanPlaceDep
const debug = require('./debug.js')
const Link = require('./link.js')
const gatherDepSet = require('./gather-dep-set.js')
const peerEntrySets = require('./peer-entry-sets.js')
class PlaceDep {
constructor (options) {
this.auditReport = options.auditReport
this.dep = options.dep
this.edge = options.edge
this.explicitRequest = options.explicitRequest
this.force = options.force
this.installLinks = options.installLinks
this.installStrategy = options.installStrategy
this.legacyPeerDeps = options.legacyPeerDeps
this.parent = options.parent || null
this.preferDedupe = options.preferDedupe
this.strictPeerDeps = options.strictPeerDeps
this.updateNames = options.updateNames
this.canPlace = null
this.canPlaceSelf = null
// XXX this only appears to be used by tests
this.checks = new Map()
this.children = []
this.needEvaluation = new Set()
this.peerConflict = null
this.placed = null
this.target = null
this.current = this.edge.to
this.name = this.edge.name
this.top = this.parent?.top || this
// nothing to do if the edge is fine as it is
if (this.edge.to &&
!this.edge.error &&
!this.explicitRequest &&
!this.updateNames.includes(this.edge.name) &&
!this.auditReport?.isVulnerable(this.edge.to)) {
return
}
// walk up the tree until we hit either a top/root node, or a place
// where the dep is not a peer dep.
const start = this.getStartNode()
for (const target of start.ancestry()) {
// if the current location has a peerDep on it, then we can't place here
// this is pretty rare to hit, since we always prefer deduping peers,
// and the getStartNode will start us out above any peers from the
// thing that depends on it. but we could hit it with something like:
//
// a -> (b@1, c@1)
// +-- c@1
// +-- b -> PEEROPTIONAL(v) (c@2)
// +-- c@2 -> (v)
//
// So we check if we can place v under c@2, that's fine.
// Then we check under b, and can't, because of the optional peer dep.
// but we CAN place it under a, so the correct thing to do is keep
// walking up the tree.
const targetEdge = target.edgesOut.get(this.edge.name)
if (!target.isTop && targetEdge && targetEdge.peer) {
continue
}
const cpd = new CanPlaceDep({
dep: this.dep,
edge: this.edge,
// note: this sets the parent's canPlace as the parent of this
// canPlace, but it does NOT add this canPlace to the parent's
// children. This way, we can know that it's a peer dep, and
// get the top edge easily, while still maintaining the
// tree of checks that factored into the original decision.
parent: this.parent && this.parent.canPlace,
target,
preferDedupe: this.preferDedupe,
explicitRequest: this.explicitRequest,
})
this.checks.set(target, cpd)
// It's possible that a "conflict" is a conflict among the *peers* of
// a given node we're trying to place, but there actually is no current
// node. Eg,
// root -> (a, b)
// a -> PEER(c)
// b -> PEER(d)
// d -> PEER(c@2)
// We place (a), and get a peer of (c) along with it.
// then we try to place (b), and get CONFLICT in the check, because
// of the conflicting peer from (b)->(d)->(c@2). In that case, we
// should treat (b) and (d) as OK, and place them in the last place
// where they did not themselves conflict, and skip c@2 if conflict
// is ok by virtue of being forced or not ours and not strict.
if (cpd.canPlaceSelf !== CONFLICT) {
this.canPlaceSelf = cpd
}
// we found a place this can go, along with all its peer friends.
// we break when we get the first conflict
if (cpd.canPlace !== CONFLICT) {
this.canPlace = cpd
} else {
break
}
// if it's a load failure, just plop it in the first place attempted,
// since we're going to crash the build or prune it out anyway.
// but, this will frequently NOT be a successful canPlace, because
// it'll have no version or other information.
if (this.dep.errors.length) {
break
}
// nest packages like npm v1 and v2
// very disk-inefficient
if (this.installStrategy === 'nested') {
break
}
// when installing globally, or just in global style, we never place
// deps above the first level.
if (this.installStrategy === 'shallow') {
const rp = target.resolveParent
if (rp && rp.isProjectRoot) {
break
}
}
}
// if we can't find a target, that means that the last place checked,
// and all the places before it, had a conflict.
if (!this.canPlace) {
// if not forced, and it's our dep, or strictPeerDeps is set, then
// this is an ERESOLVE error.
if (!this.force && (this.isMine || this.strictPeerDeps)) {
return this.failPeerConflict()
}
// ok! we're gonna allow the conflict, but we should still warn
// if we have a current, then we treat CONFLICT as a KEEP.
// otherwise, we just skip it. Only warn on the one that actually
// could not be placed somewhere.
if (!this.canPlaceSelf) {
this.warnPeerConflict()
return
}
this.canPlace = this.canPlaceSelf
}
// now we have a target, a tree of CanPlaceDep results for the peer group,
// and we are ready to go
/* istanbul ignore next */
if (!this.canPlace) {
debug(() => {
throw new Error('canPlace not set, but trying to place in tree')
})
return
}
const { target } = this.canPlace
log.silly(
'placeDep',
target.location || 'ROOT',
`${this.dep.name}@${this.dep.version}`,
this.canPlace.description,
`for: ${this.edge.from.package._id || this.edge.from.location}`,
`want: ${redact(this.edge.spec || '*')}`
)
const placementType = this.canPlace.canPlace === CONFLICT
? this.canPlace.canPlaceSelf
: this.canPlace.canPlace
// if we're placing in the tree with --force, we can get here even though
// it's a conflict. Treat it as a KEEP, but warn and move on.
if (placementType === KEEP) {
// this was a peerConflicted peer dep
if (this.edge.peer && !this.edge.valid) {
this.warnPeerConflict()
}
// if we get a KEEP in an update scenario, then we MAY have something
// already duplicating this unnecessarily! For example:
// ```
// root (dep: y@1)
// +-- x (dep: y@1.1)
// | +-- y@1.1.0 (replacing with 1.1.2, got KEEP at the root)
// +-- y@1.1.2 (updated already from 1.0.0)
// ```
// Now say we do `reify({update:['y']})`, and the latest version is
// 1.1.2, which we now have in the root. We'll try to place y@1.1.2
// first in x, then in the root, ending with KEEP, because we already
// have it. In that case, we ought to REMOVE the nm/x/nm/y node, because
// it is an unnecessary duplicate.
this.pruneDedupable(target)
return
}
// we were told to place it here in the target, so either it does not
// already exist in the tree, OR it's shadowed.
// handle otherwise unresolvable dependency nesting loops by
// creating a symbolic link
// a1 -> b1 -> a2 -> b2 -> a1 -> ...
// instead of nesting forever, when the loop occurs, create
// a symbolic link to the earlier instance
for (let p = target; p; p = p.resolveParent) {
if (p.matches(this.dep) && !p.isTop) {
this.placed = new Link({ parent: target, target: p })
return
}
}
// XXX if we are replacing SOME of a peer entry group, we will need to
// remove any that are not being replaced and will now be invalid, and
// re-evaluate them deeper into the tree.
const virtualRoot = this.dep.parent
this.placed = new this.dep.constructor({
name: this.dep.name,
pkg: this.dep.package,
resolved: this.dep.resolved,
integrity: this.dep.integrity,
installLinks: this.installLinks,
legacyPeerDeps: this.legacyPeerDeps,
error: this.dep.errors[0],
...(this.dep.overrides ? { overrides: this.dep.overrides } : {}),
...(this.dep.isLink ? { target: this.dep.target, realpath: this.dep.realpath } : {}),
})
this.oldDep = target.children.get(this.name)
if (this.oldDep) {
this.replaceOldDep()
} else {
this.placed.parent = target
}
// if it's a peerConflicted peer dep, warn about it
if (this.edge.peer && !this.placed.satisfies(this.edge)) {
this.warnPeerConflict()
}
// If the edge is not an error, then we're updating something, and
// MAY end up putting a better/identical node further up the tree in
// a way that causes an unnecessary duplication. If so, remove the
// now-unnecessary node.
if (this.edge.valid && this.edge.to && this.edge.to !== this.placed) {
this.pruneDedupable(this.edge.to, false)
}
// in case we just made some duplicates that can be removed,
// prune anything deeper in the tree that can be replaced by this
for (const node of target.root.inventory.query('name', this.name)) {
if (node.isDescendantOf(target) && !node.isTop) {
this.pruneDedupable(node, false)
// only walk the direct children of the ones we kept
if (node.root === target.root) {
for (const kid of node.children.values()) {
this.pruneDedupable(kid, false)
}
}
}
}
// also place its unmet or invalid peer deps at this location
// loop through any peer deps from the thing we just placed, and place
// those ones as well. it's safe to do this with the virtual nodes,
// because we're copying rather than moving them out of the virtual root,
// otherwise they'd be gone and the peer set would change throughout
// this loop.
for (const peerEdge of this.placed.edgesOut.values()) {
if (peerEdge.valid || !peerEdge.peer || peerEdge.peerConflicted) {
continue
}
const peer = virtualRoot.children.get(peerEdge.name)
// Note: if the virtualRoot *doesn't* have the peer, then that means
// it's an optional peer dep. If it's not being properly met (ie,
// peerEdge.valid is false), then this is likely heading for an
// ERESOLVE error, unless it can walk further up the tree.
if (!peer) {
continue
}
// peerConflicted peerEdge, just accept what's there already
if (!peer.satisfies(peerEdge)) {
continue
}
this.children.push(new PlaceDep({
auditReport: this.auditReport,
explicitRequest: this.explicitRequest,
force: this.force,
installLinks: this.installLinks,
installStrategy: this.installStrategy,
legacyPeerDeps: this.legacyPeerDeps,
preferDedupe: this.preferDedupe,
strictPeerDeps: this.strictPeerDeps,
updateNames: this.updateName,
parent: this,
dep: peer,
node: this.placed,
edge: peerEdge,
}))
}
}
replaceOldDep () {
const target = this.oldDep.parent
// XXX handle replacing an entire peer group?
// what about cases where we need to push some other peer groups deeper
// into the tree? all the tree updating should be done here, and track
// all the things that we add and remove, so that we can know what
// to re-evaluate.
// if we're replacing, we should also remove any nodes for edges that
// are now invalid, and where this (or its deps) is the only dependent,
// and also recurse on that pruning. Otherwise leaving that dep node
// around can result in spurious conflicts pushing nodes deeper into
// the tree than needed in the case of cycles that will be removed
// later anyway.
const oldDeps = []
for (const [name, edge] of this.oldDep.edgesOut.entries()) {
if (!this.placed.edgesOut.has(name) && edge.to) {
oldDeps.push(...gatherDepSet([edge.to], e => e.to !== edge.to))
}
}
// gather all peer edgesIn which are at this level, and will not be
// satisfied by the new dependency. Those are the peer sets that need
// to be either warned about (if they cannot go deeper), or removed and
// re-placed (if they can).
const prunePeerSets = []
for (const edge of this.oldDep.edgesIn) {
if (this.placed.satisfies(edge) ||
!edge.peer ||
edge.from.parent !== target ||
edge.peerConflicted) {
// not a peer dep, not invalid, or not from this level, so it's fine
// to just let it re-evaluate as a problemEdge later, or let it be
// satisfied by the new dep being placed.
continue
}
for (const entryEdge of peerEntrySets(edge.from).keys()) {
// either this one needs to be pruned and re-evaluated, or marked
// as peerConflicted and warned about. If the entryEdge comes in from
// the root or a workspace, then we have to leave it alone, and in that
// case, it will have already warned or crashed by getting to this point
const entryNode = entryEdge.to
const deepestTarget = deepestNestingTarget(entryNode)
if (deepestTarget !== target &&
!(entryEdge.from.isProjectRoot || entryEdge.from.isWorkspace)) {
prunePeerSets.push(...gatherDepSet([entryNode], e => {
return e.to !== entryNode && !e.peerConflicted
}))
} else {
this.warnPeerConflict(edge, this.dep)
}
}
}
this.placed.replace(this.oldDep)
this.pruneForReplacement(this.placed, oldDeps)
for (const dep of prunePeerSets) {
for (const edge of dep.edgesIn) {
this.needEvaluation.add(edge.from)
}
dep.root = null
}
}
pruneForReplacement (node, oldDeps) {
// gather up all the now-invalid/extraneous edgesOut, as long as they are
// only depended upon by the old node/deps
const invalidDeps = new Set([...node.edgesOut.values()]
.filter(e => e.to && !e.valid).map(e => e.to))
for (const dep of oldDeps) {
const set = gatherDepSet([dep], e => e.to !== dep && e.valid)
for (const dep of set) {
invalidDeps.add(dep)
}
}
// ignore dependency edges from the node being replaced, but
// otherwise filter the set down to just the set with no
// dependencies from outside the set, except the node in question.
const deps = gatherDepSet(invalidDeps, edge =>
edge.from !== node && edge.to !== node && edge.valid)
// now just delete whatever's left, because it's junk
for (const dep of deps) {
dep.root = null
}
}
// prune all the nodes in a branch of the tree that can be safely removed
// This is only the most basic duplication detection; it finds if there
// is another satisfying node further up the tree, and if so, dedupes.
// Even if installStrategy is nested, we do this amount of deduplication.
pruneDedupable (node, descend = true) {
if (node.canDedupe(this.preferDedupe, this.explicitRequest)) {
// gather up all deps that have no valid edges in from outside
// the dep set, except for this node we're deduping, so that we
// also prune deps that would be made extraneous.
const deps = gatherDepSet([node], e => e.to !== node && e.valid)
for (const node of deps) {
node.root = null
}
return
}
if (descend) {
// sort these so that they're deterministically ordered
// otherwise, resulting tree shape is dependent on the order
// in which they happened to be resolved.
const nodeSort = (a, b) => localeCompare(a.location, b.location)
const children = [...node.children.values()].sort(nodeSort)
for (const child of children) {
this.pruneDedupable(child)
}
const fsChildren = [...node.fsChildren].sort(nodeSort)
for (const topNode of fsChildren) {
const children = [...topNode.children.values()].sort(nodeSort)
for (const child of children) {
this.pruneDedupable(child)
}
}
}
}
get isMine () {
const { edge } = this.top
const { from: node } = edge
if (node.isWorkspace || node.isProjectRoot) {
return true
}
if (!edge.peer) {
return false
}
// re-entry case. check if any non-peer edges come from the project,
// or any entryEdges on peer groups are from the root.
let hasPeerEdges = false
for (const edge of node.edgesIn) {
if (edge.peer) {
hasPeerEdges = true
continue
}
if (edge.from.isWorkspace || edge.from.isProjectRoot) {
return true
}
}
if (hasPeerEdges) {
for (const edge of peerEntrySets(node).keys()) {
if (edge.from.isWorkspace || edge.from.isProjectRoot) {
return true
}
}
}
return false
}
warnPeerConflict (edge, dep) {
edge = edge || this.edge
dep = dep || this.dep
edge.peerConflicted = true
const expl = this.explainPeerConflict(edge, dep)
log.warn('ERESOLVE', 'overriding peer dependency', expl)
}
failPeerConflict (edge, dep) {
edge = edge || this.top.edge
dep = dep || this.top.dep
const expl = this.explainPeerConflict(edge, dep)
throw Object.assign(new Error('could not resolve'), expl)
}
explainPeerConflict (edge, dep) {
const { from: node } = edge
const curNode = node.resolve(edge.name)
// XXX decorate more with this.canPlace and this.canPlaceSelf,
// this.checks, this.children, walk over conflicted peers, etc.
const expl = {
code: 'ERESOLVE',
edge: edge.explain(),
dep: dep.explain(edge),
force: this.force,
isMine: this.isMine,
strictPeerDeps: this.strictPeerDeps,
}
if (this.parent) {
// this is the conflicted peer
expl.current = curNode && curNode.explain(edge)
expl.peerConflict = this.current && this.current.explain(this.edge)
} else {
expl.current = curNode && curNode.explain()
if (this.canPlaceSelf && this.canPlaceSelf.canPlaceSelf !== CONFLICT) {
// failed while checking for a child dep
const cps = this.canPlaceSelf
for (const peer of cps.conflictChildren) {
if (peer.current) {
expl.peerConflict = {
current: peer.current.explain(),
peer: peer.dep.explain(peer.edge),
}
break
}
}
} else {
expl.peerConflict = {
current: this.current && this.current.explain(),
peer: this.dep.explain(this.edge),
}
}
}
return expl
}
getStartNode () {
// if we are a peer, then we MUST be at least as shallow as the peer
// dependent
const from = this.parent?.getStartNode() || this.edge.from
return deepestNestingTarget(from, this.name)
}
// XXX this only appears to be used by tests
get allChildren () {
const set = new Set(this.children)
for (const child of set) {
for (const grandchild of child.children) {
set.add(grandchild)
}
}
return [...set]
}
}
module.exports = PlaceDep
@@ -0,0 +1,198 @@
// helper function to output a clearer visualization
// of the current node and its descendents
const localeCompare = require('@isaacs/string-locale-compare')('en')
const util = require('node:util')
const relpath = require('./relpath.js')
class ArboristNode {
constructor (tree, path) {
this.name = tree.name
if (tree.packageName && tree.packageName !== this.name) {
this.packageName = tree.packageName
}
if (tree.version) {
this.version = tree.version
}
this.location = tree.location
this.path = tree.path
if (tree.realpath !== this.path) {
this.realpath = tree.realpath
}
if (tree.resolved !== null) {
this.resolved = tree.resolved
}
if (tree.extraneous) {
this.extraneous = true
}
if (tree.dev) {
this.dev = true
}
if (tree.optional) {
this.optional = true
}
if (tree.devOptional && !tree.dev && !tree.optional) {
this.devOptional = true
}
if (tree.peer) {
this.peer = true
}
if (tree.inBundle) {
this.bundled = true
}
if (tree.inDepBundle) {
this.bundler = tree.getBundler().location
}
if (tree.isProjectRoot) {
this.isProjectRoot = true
}
if (tree.isWorkspace) {
this.isWorkspace = true
}
const bd = tree.package && tree.package.bundleDependencies
if (bd && bd.length) {
this.bundleDependencies = bd
}
if (tree.inShrinkwrap) {
this.inShrinkwrap = true
} else if (tree.hasShrinkwrap) {
this.hasShrinkwrap = true
}
if (tree.error) {
this.error = treeError(tree.error)
}
if (tree.errors && tree.errors.length) {
this.errors = tree.errors.map(treeError)
}
if (tree.overrides) {
this.overrides = new Map([...tree.overrides.ruleset.values()]
.map((override) => [override.key, override.value]))
}
// edgesOut sorted by name
if (tree.edgesOut.size) {
this.edgesOut = new Map([...tree.edgesOut.entries()]
.sort(([a], [b]) => localeCompare(a, b))
.map(([name, edge]) => [name, new EdgeOut(edge)]))
}
// edgesIn sorted by location
if (tree.edgesIn.size) {
this.edgesIn = new Set([...tree.edgesIn]
.sort((a, b) => localeCompare(a.from.location, b.from.location))
.map(edge => new EdgeIn(edge)))
}
if (tree.workspaces && tree.workspaces.size) {
this.workspaces = new Map([...tree.workspaces.entries()]
.map(([name, path]) => [name, relpath(tree.root.realpath, path)]))
}
// fsChildren sorted by path
if (tree.fsChildren.size) {
this.fsChildren = new Set([...tree.fsChildren]
.sort(({ path: a }, { path: b }) => localeCompare(a, b))
.map(tree => printableTree(tree, path)))
}
// children sorted by name
if (tree.children.size) {
this.children = new Map([...tree.children.entries()]
.sort(([a], [b]) => localeCompare(a, b))
.map(([name, tree]) => [name, printableTree(tree, path)]))
}
}
}
class ArboristVirtualNode extends ArboristNode {
constructor (tree, path) {
super(tree, path)
this.sourceReference = printableTree(tree.sourceReference, path)
}
}
class ArboristLink extends ArboristNode {
constructor (tree, path) {
super(tree, path)
this.target = printableTree(tree.target, path)
}
}
const treeError = ({ code, path }) => ({
code,
...(path ? { path } : {}),
})
// print out edges without dumping the full node all over again
// this base class will toJSON as a plain old object, but the
// util.inspect() output will be a bit cleaner
class Edge {
constructor (edge) {
this.type = edge.type
this.name = edge.name
this.spec = edge.rawSpec || '*'
if (edge.rawSpec !== edge.spec) {
this.override = edge.spec
}
if (edge.error) {
this.error = edge.error
}
if (edge.peerConflicted) {
this.peerConflicted = edge.peerConflicted
}
}
}
// don't care about 'from' for edges out
class EdgeOut extends Edge {
constructor (edge) {
super(edge)
this.to = edge.to && edge.to.location
}
[util.inspect.custom] () {
return `{ ${this.type} ${this.name}@${this.spec}${
this.override ? ` overridden:${this.override}` : ''
}${
this.to ? ' -> ' + this.to : ''
}${
this.error ? ' ' + this.error : ''
}${
this.peerConflicted ? ' peerConflicted' : ''
} }`
}
}
// don't care about 'to' for edges in
class EdgeIn extends Edge {
constructor (edge) {
super(edge)
this.from = edge.from && edge.from.location
}
[util.inspect.custom] () {
return `{ ${this.from || '""'} ${this.type} ${this.name}@${this.spec}${
this.error ? ' ' + this.error : ''
}${
this.peerConflicted ? ' peerConflicted' : ''
} }`
}
}
const printableTree = (tree, path = []) => {
if (!tree) {
return tree
}
const Cls = tree.isLink ? ArboristLink
: tree.sourceReference ? ArboristVirtualNode
: ArboristNode
if (path.includes(tree)) {
const obj = Object.create(Cls.prototype)
return Object.assign(obj, { location: tree.location })
}
path.push(tree)
return new Cls(tree, path)
}
module.exports = printableTree
@@ -0,0 +1,958 @@
'use strict'
const { resolve } = require('node:path')
const { parser, arrayDelimiter } = require('@npmcli/query')
const localeCompare = require('@isaacs/string-locale-compare')('en')
const { log } = require('proc-log')
const { minimatch } = require('minimatch')
const npa = require('npm-package-arg')
const pacote = require('pacote')
const semver = require('semver')
const npmFetch = require('npm-registry-fetch')
// handle results for parsed query asts, results are stored in a map that has a
// key that points to each ast selector node and stores the resulting array of
// arborist nodes as its value, that is essential to how we handle multiple
// query selectors, e.g: `#a, #b, #c` <- 3 diff ast selector nodes
class Results {
#currentAstSelector
#initialItems
#inventory
#outdatedCache = new Map()
#vulnCache
#pendingCombinator
#results = new Map()
#targetNode
constructor (opts) {
this.#currentAstSelector = opts.rootAstNode.nodes[0]
this.#inventory = opts.inventory
this.#initialItems = opts.initialItems
this.#vulnCache = opts.vulnCache
this.#targetNode = opts.targetNode
this.currentResults = this.#initialItems
// We get this when first called and need to pass it to pacote
this.flatOptions = opts.flatOptions || {}
// reset by rootAstNode walker
this.currentAstNode = opts.rootAstNode
}
get currentResults () {
return this.#results.get(this.#currentAstSelector)
}
set currentResults (value) {
this.#results.set(this.#currentAstSelector, value)
}
// retrieves the initial items to which start the filtering / matching
// for most of the different types of recognized ast nodes, e.g: class (aka
// depType), id, *, etc in different contexts we need to start with the
// current list of filtered results, for example a query for `.workspace`
// actually means the same as `*.workspace` so we want to start with the full
// inventory if that's the first ast node we're reading but if it appears in
// the middle of a query it should respect the previous filtered results,
// combinators are a special case in which we always want to have the
// complete inventory list in order to use the left-hand side ast node as a
// filter combined with the element on its right-hand side
get initialItems () {
const firstParsed =
(this.currentAstNode.parent.nodes[0] === this.currentAstNode) &&
(this.currentAstNode.parent.parent.type === 'root')
if (firstParsed) {
return this.#initialItems
}
if (this.currentAstNode.prev().type === 'combinator') {
return this.#inventory
}
return this.currentResults
}
// combinators need information about previously filtered items along
// with info of the items parsed / retrieved from the selector right
// past the combinator, for this reason combinators are stored and
// only ran as the last part of each selector logic
processPendingCombinator (nextResults) {
if (this.#pendingCombinator) {
const res = this.#pendingCombinator(this.currentResults, nextResults)
this.#pendingCombinator = null
this.currentResults = res
} else {
this.currentResults = nextResults
}
}
// when collecting results to a root astNode, we traverse the list of child
// selector nodes and collect all of their resulting arborist nodes into a
// single/flat Set of items, this ensures we also deduplicate items
collect (rootAstNode) {
return new Set(rootAstNode.nodes.flatMap(n => this.#results.get(n)))
}
// selector types map to the '.type' property of the ast nodes via `${astNode.type}Type`
//
// attribute selector [name=value], etc
attributeType () {
const nextResults = this.initialItems.filter(node =>
attributeMatch(this.currentAstNode, node.package)
)
this.processPendingCombinator(nextResults)
}
// dependency type selector (i.e. .prod, .dev, etc)
// css calls this class, we interpret is as dependency type
classType () {
const depTypeFn = depTypes[String(this.currentAstNode)]
if (!depTypeFn) {
throw Object.assign(
new Error(`\`${String(this.currentAstNode)}\` is not a supported dependency type.`),
{ code: 'EQUERYNODEPTYPE' }
)
}
const nextResults = depTypeFn(this.initialItems)
this.processPendingCombinator(nextResults)
}
// combinators (i.e. '>', ' ', '~')
combinatorType () {
this.#pendingCombinator = combinators[String(this.currentAstNode)]
}
// name selectors (i.e. #foo)
// css calls this id, we interpret it as name
idType () {
const name = this.currentAstNode.value
const nextResults = this.initialItems.filter(node =>
(name === node.name) || (name === node.package.name)
)
this.processPendingCombinator(nextResults)
}
// pseudo selectors (prefixed with :)
async pseudoType () {
const pseudoFn = `${this.currentAstNode.value.slice(1)}Pseudo`
if (!this[pseudoFn]) {
throw Object.assign(
new Error(`\`${this.currentAstNode.value
}\` is not a supported pseudo selector.`),
{ code: 'EQUERYNOPSEUDO' }
)
}
const nextResults = await this[pseudoFn]()
this.processPendingCombinator(nextResults)
}
selectorType () {
this.#currentAstSelector = this.currentAstNode
// starts a new array in which resulting items
// can be stored for each given ast selector
if (!this.currentResults) {
this.currentResults = []
}
}
universalType () {
this.processPendingCombinator(this.initialItems)
}
// pseudo selectors map to the 'value' property of the pseudo selectors in the ast nodes
// via selectors via `${value.slice(1)}Pseudo`
attrPseudo () {
const { lookupProperties, attributeMatcher } = this.currentAstNode
return this.initialItems.filter(node => {
let objs = [node.package]
for (const prop of lookupProperties) {
// if an isArray symbol is found that means we'll need to iterate
// over the previous found array to basically make sure we traverse
// all its indexes testing for possible objects that may eventually
// hold more keys specified in a selector
if (prop === arrayDelimiter) {
objs = objs.flat()
continue
}
// otherwise just maps all currently found objs
// to the next prop from the lookup properties list,
// filters out any empty key lookup
objs = objs.flatMap(obj => obj[prop] || [])
// in case there's no property found in the lookup
// just filters that item out
const noAttr = objs.every(obj => !obj)
if (noAttr) {
return false
}
}
// if any of the potential object matches
// that item should be in the final result
return objs.some(obj => attributeMatch(attributeMatcher, obj))
})
}
emptyPseudo () {
return this.initialItems.filter(node => node.edgesOut.size === 0)
}
extraneousPseudo () {
return this.initialItems.filter(node => node.extraneous)
}
async hasPseudo () {
const found = []
for (const item of this.initialItems) {
// This is the one time initialItems differs from inventory
const res = await retrieveNodesFromParsedAst({
flatOptions: this.flatOptions,
initialItems: [item],
inventory: this.#inventory,
rootAstNode: this.currentAstNode.nestedNode,
targetNode: item,
vulnCache: this.#vulnCache,
})
if (res.size > 0) {
found.push(item)
}
}
return found
}
invalidPseudo () {
const found = []
for (const node of this.initialItems) {
for (const edge of node.edgesIn) {
if (edge.invalid) {
found.push(node)
break
}
}
}
return found
}
async isPseudo () {
const res = await retrieveNodesFromParsedAst({
flatOptions: this.flatOptions,
initialItems: this.initialItems,
inventory: this.#inventory,
rootAstNode: this.currentAstNode.nestedNode,
targetNode: this.currentAstNode,
vulnCache: this.#vulnCache,
})
return [...res]
}
linkPseudo () {
return this.initialItems.filter(node => node.isLink || (node.isTop && !node.isRoot))
}
missingPseudo () {
return this.#inventory.reduce((res, node) => {
for (const edge of node.edgesOut.values()) {
if (edge.missing) {
const pkg = { name: edge.name, version: edge.spec }
const item = new this.#targetNode.constructor({ pkg })
item.queryContext = {
missing: true,
}
item.edgesIn = new Set([edge])
res.push(item)
}
}
return res
}, [])
}
async notPseudo () {
const res = await retrieveNodesFromParsedAst({
flatOptions: this.flatOptions,
initialItems: this.initialItems,
inventory: this.#inventory,
rootAstNode: this.currentAstNode.nestedNode,
targetNode: this.currentAstNode,
vulnCache: this.#vulnCache,
})
const internalSelector = new Set(res)
return this.initialItems.filter(node =>
!internalSelector.has(node))
}
overriddenPseudo () {
return this.initialItems.filter(node => node.overridden)
}
pathPseudo () {
return this.initialItems.filter(node => {
if (!this.currentAstNode.pathValue) {
return true
}
return minimatch(
node.realpath.replace(/\\+/g, '/'),
resolve(node.root.realpath, this.currentAstNode.pathValue).replace(/\\+/g, '/')
)
})
}
privatePseudo () {
return this.initialItems.filter(node => node.package.private)
}
rootPseudo () {
return this.initialItems.filter(node => node === this.#targetNode.root)
}
scopePseudo () {
return this.initialItems.filter(node => node === this.#targetNode)
}
semverPseudo () {
const {
attributeMatcher,
lookupProperties,
semverFunc = 'infer',
semverValue,
} = this.currentAstNode
const { qualifiedAttribute } = attributeMatcher
if (!semverValue) {
// DEPRECATED: remove this warning and throw an error as part of @npmcli/arborist@6
log.warn('query', 'usage of :semver() with no parameters is deprecated')
return this.initialItems
}
if (!semver.valid(semverValue) && !semver.validRange(semverValue)) {
throw Object.assign(
new Error(`\`${semverValue}\` is not a valid semver version or range`),
{ code: 'EQUERYINVALIDSEMVER' })
}
const valueIsVersion = !!semver.valid(semverValue)
const nodeMatches = (node, obj) => {
// if we already have an operator, the user provided some test as part of the selector
// we evaluate that first because if it fails we don't want this node anyway
if (attributeMatcher.operator) {
if (!attributeMatch(attributeMatcher, obj)) {
// if the initial operator doesn't match, we're done
return false
}
}
const attrValue = obj[qualifiedAttribute]
// both valid and validRange return null for undefined, so this will skip both nodes that
// do not have the attribute defined as well as those where the attribute value is invalid
// and those where the value from the package.json is not a string
if ((!semver.valid(attrValue) && !semver.validRange(attrValue)) ||
typeof attrValue !== 'string') {
return false
}
const attrIsVersion = !!semver.valid(attrValue)
let actualFunc = semverFunc
// if we're asked to infer, we examine outputs to make a best guess
if (actualFunc === 'infer') {
if (valueIsVersion && attrIsVersion) {
// two versions -> semver.eq
actualFunc = 'eq'
} else if (!valueIsVersion && !attrIsVersion) {
// two ranges -> semver.intersects
actualFunc = 'intersects'
} else {
// anything else -> semver.satisfies
actualFunc = 'satisfies'
}
}
if (['eq', 'neq', 'gt', 'gte', 'lt', 'lte'].includes(actualFunc)) {
// both sides must be versions, but one is not
if (!valueIsVersion || !attrIsVersion) {
return false
}
return semver[actualFunc](attrValue, semverValue)
} else if (['gtr', 'ltr', 'satisfies'].includes(actualFunc)) {
// at least one side must be a version, but neither is
if (!valueIsVersion && !attrIsVersion) {
return false
}
return valueIsVersion
? semver[actualFunc](semverValue, attrValue)
: semver[actualFunc](attrValue, semverValue)
} else if (['intersects', 'subset'].includes(actualFunc)) {
// these accept two ranges and since a version is also a range, anything goes
return semver[actualFunc](attrValue, semverValue)
} else {
// user provided a function we don't know about, throw an error
throw Object.assign(new Error(`\`semver.${actualFunc}\` is not a supported operator.`),
{ code: 'EQUERYINVALIDOPERATOR' })
}
}
return this.initialItems.filter((node) => {
// no lookupProperties just means its a top level property, see if it matches
if (!lookupProperties.length) {
return nodeMatches(node, node.package)
}
// this code is mostly duplicated from attrPseudo to traverse into the package until we get
// to our deepest requested object
let objs = [node.package]
for (const prop of lookupProperties) {
if (prop === arrayDelimiter) {
objs = objs.flat()
continue
}
objs = objs.flatMap(obj => obj[prop] || [])
const noAttr = objs.every(obj => !obj)
if (noAttr) {
return false
}
return objs.some(obj => nodeMatches(node, obj))
}
})
}
typePseudo () {
if (!this.currentAstNode.typeValue) {
return this.initialItems
}
// TODO this differs subtly with `:type()` because it now iterates on edgesIn, which means extraneous deps won't show up
// note how "@npmcli/abbrev@2.0.0-beta.45" is in the `:type()` results in the test but not in any of the other results.
return this.initialItems
.flatMap(node => {
const found = []
const { typeValue } = this.currentAstNode
for (const edge of node.edgesIn) {
const parsedArg = npa(`${edge.name}@${edge.spec}`)
if (typeValue === 'registry') {
if (parsedArg.registry) {
found.push(edge.to)
}
} else if (parsedArg.type === typeValue) {
found.push(edge.to)
}
}
return found
})
}
dedupedPseudo () {
return this.initialItems.filter(node => node.target.edgesIn.size > 1)
}
async vulnPseudo () {
if (!this.initialItems.length) {
return this.initialItems
}
if (!this.#vulnCache) {
const packages = {}
// We have to map the items twice, once to get the request, and a second time to filter out the results of that request
this.initialItems.map((node) => {
if (node.isProjectRoot || node.package.private) {
return
}
if (!packages[node.name]) {
packages[node.name] = []
}
if (!packages[node.name].includes(node.version)) {
packages[node.name].push(node.version)
}
})
const res = await npmFetch('/-/npm/v1/security/advisories/bulk', {
...this.flatOptions,
registry: this.flatOptions.auditRegistry || this.flatOptions.registry,
method: 'POST',
gzip: true,
body: packages,
})
this.#vulnCache = await res.json()
}
const advisories = this.#vulnCache
const { vulns } = this.currentAstNode
return this.initialItems.filter(item => {
const vulnerable = advisories[item.name]?.filter(advisory => {
// This could be for another version of this package elsewhere in the tree
if (!semver.intersects(advisory.vulnerable_versions, item.version)) {
return false
}
if (!vulns) {
return true
}
// vulns are OR with each other, if any one matches we're done
for (const vuln of vulns) {
if (vuln.severity && !vuln.severity.includes('*')) {
if (!vuln.severity.includes(advisory.severity)) {
continue
}
}
if (vuln?.cwe) {
// * is special, it means "has a cwe"
if (vuln.cwe.includes('*')) {
if (!advisory.cwe.length) {
continue
}
} else if (!vuln.cwe.every(cwe => advisory.cwe.includes(`CWE-${cwe}`))) {
continue
}
}
return true
}
})
if (vulnerable?.length) {
item.queryContext = {
advisories: vulnerable,
}
return true
}
return false
})
}
async outdatedPseudo () {
const { outdatedKind = 'any' } = this.currentAstNode
// filter the initialItems
// NOTE: this uses a Promise.all around a map without in-line concurrency handling
// since the only async action taken is retrieving the packument, which is limited
// based on the max-sockets config in make-fetch-happen
const initialResults = await Promise.all(this.initialItems.map(async (node) => {
// the root can't be outdated, skip it
if (node.isProjectRoot) {
return false
}
// private packages can't be published, skip them
if (node.package.private) {
return false
}
// we cache the promise representing the full versions list, this helps reduce the
// number of requests we send by keeping population of the cache in a single tick
// making it less likely that multiple requests for the same package will be inflight
if (!this.#outdatedCache.has(node.name)) {
this.#outdatedCache.set(node.name, getPackageVersions(node.name, this.flatOptions))
}
const availableVersions = await this.#outdatedCache.get(node.name)
// we attach _all_ versions to the queryContext to allow consumers to do their own
// filtering and comparisons
node.queryContext.versions = availableVersions
// next we further reduce the set to versions that are greater than the current one
const greaterVersions = availableVersions.filter((available) => {
return semver.gt(available, node.version)
})
// no newer versions than the current one, drop this node from the result set
if (!greaterVersions.length) {
return false
}
// if we got here, we know that newer versions exist, if the kind is 'any' we're done
if (outdatedKind === 'any') {
return node
}
// look for newer versions that differ from current by a specific part of the semver version
if (['major', 'minor', 'patch'].includes(outdatedKind)) {
// filter the versions greater than our current one based on semver.diff
const filteredVersions = greaterVersions.filter((version) => {
return semver.diff(node.version, version) === outdatedKind
})
// no available versions are of the correct diff type
if (!filteredVersions.length) {
return false
}
return node
}
// look for newer versions that satisfy at least one edgeIn to this node
if (outdatedKind === 'in-range') {
const inRangeContext = []
for (const edge of node.edgesIn) {
const inRangeVersions = greaterVersions.filter((version) => {
return semver.satisfies(version, edge.spec)
})
// this edge has no in-range candidates, just move on
if (!inRangeVersions.length) {
continue
}
inRangeContext.push({
from: edge.from.location,
versions: inRangeVersions,
})
}
// if we didn't find at least one match, drop this node
if (!inRangeContext.length) {
return false
}
// now add to the context each version that is in-range for each edgeIn
node.queryContext.outdated = {
...node.queryContext.outdated,
inRange: inRangeContext,
}
return node
}
// look for newer versions that _do not_ satisfy at least one edgeIn
if (outdatedKind === 'out-of-range') {
const outOfRangeContext = []
for (const edge of node.edgesIn) {
const outOfRangeVersions = greaterVersions.filter((version) => {
return !semver.satisfies(version, edge.spec)
})
// this edge has no out-of-range candidates, skip it
if (!outOfRangeVersions.length) {
continue
}
outOfRangeContext.push({
from: edge.from.location,
versions: outOfRangeVersions,
})
}
// if we didn't add at least one thing to the context, this node is not a match
if (!outOfRangeContext.length) {
return false
}
// attach the out-of-range context to the node
node.queryContext.outdated = {
...node.queryContext.outdated,
outOfRange: outOfRangeContext,
}
return node
}
// any other outdatedKind is unknown and will never match
return false
}))
// return an array with the holes for non-matching nodes removed
return initialResults.filter(Boolean)
}
}
// operators for attribute selectors
const attributeOperators = {
// attribute value is equivalent
'=' ({ attr, value }) {
return attr === value
},
// attribute value contains word
'~=' ({ attr, value }) {
return (attr.match(/\w+/g) || []).includes(value)
},
// attribute value contains string
'*=' ({ attr, value }) {
return attr.includes(value)
},
// attribute value is equal or starts with
'|=' ({ attr, value }) {
return attr.startsWith(`${value}-`)
},
// attribute value starts with
'^=' ({ attr, value }) {
return attr.startsWith(value)
},
// attribute value ends with
'$=' ({ attr, value }) {
return attr.endsWith(value)
},
}
const attributeOperator = ({ attr, value, insensitive, operator }) => {
if (typeof attr === 'number') {
attr = String(attr)
}
if (typeof attr !== 'string') {
// It's an object or an array, bail
return false
}
if (insensitive) {
attr = attr.toLowerCase()
}
return attributeOperators[operator]({
attr,
insensitive,
value,
})
}
const attributeMatch = (matcher, obj) => {
const insensitive = !!matcher.insensitive
const operator = matcher.operator || ''
const attribute = matcher.qualifiedAttribute
let value = matcher.value || ''
// return early if checking existence
if (operator === '') {
return Boolean(obj[attribute])
}
if (insensitive) {
value = value.toLowerCase()
}
// in case the current object is an array
// then we try to match every item in the array
if (Array.isArray(obj[attribute])) {
return obj[attribute].find((i, index) => {
const attr = obj[attribute][index] || ''
return attributeOperator({ attr, value, insensitive, operator })
})
} else {
const attr = obj[attribute] || ''
return attributeOperator({ attr, value, insensitive, operator })
}
}
const edgeIsType = (node, type, seen = new Set()) => {
for (const edgeIn of node.edgesIn) {
// TODO Need a test with an infinite loop
if (seen.has(edgeIn)) {
continue
}
seen.add(edgeIn)
if (edgeIn.type === type || edgeIn.from[type] || edgeIsType(edgeIn.from, type, seen)) {
return true
}
}
return false
}
const filterByType = (nodes, type) => {
const found = []
for (const node of nodes) {
if (node[type] || edgeIsType(node, type)) {
found.push(node)
}
}
return found
}
const depTypes = {
// dependency
'.prod' (prevResults) {
const found = []
for (const node of prevResults) {
if (!node.dev) {
found.push(node)
}
}
return found
},
// devDependency
'.dev' (prevResults) {
return filterByType(prevResults, 'dev')
},
// optionalDependency
'.optional' (prevResults) {
return filterByType(prevResults, 'optional')
},
// peerDependency
'.peer' (prevResults) {
return filterByType(prevResults, 'peer')
},
// workspace
'.workspace' (prevResults) {
return prevResults.filter(node => node.isWorkspace)
},
// bundledDependency
'.bundled' (prevResults) {
return prevResults.filter(node => node.inBundle)
},
}
// checks if a given node has a direct parent in any of the nodes provided in
// the compare nodes array
const hasParent = (node, compareNodes) => {
// All it takes is one so we loop and return on the first hit
for (let compareNode of compareNodes) {
if (compareNode.isLink) {
compareNode = compareNode.target
}
// Follows logical parent for link ancestors (e.g. workspaces whose target lives outside node_modules).
// Only match if the node has a link whose parent is the compareNode. Without this check, nodes deep in the store (linked strategy) would incorrectly match as children of root via their fsParent chain.
if (node.isTop && (node.resolveParent === compareNode)) {
for (const link of node.linksIn) {
if (link.parent === compareNode) {
return true
}
}
}
// follows edges-in to check if they match a possible parent
for (const edge of node.edgesIn) {
if (edge && edge.from === compareNode) {
return true
}
}
}
return false
}
// checks if a given node is a descendant of any of the nodes provided in the
// compareNodes array
const hasAscendant = (node, compareNodes, seen = new Set()) => {
// TODO (future) loop over ancestry property
if (hasParent(node, compareNodes)) {
return true
}
if (node.isTop && node.resolveParent) {
/* istanbul ignore if - investigate if linksIn check obviates need for this */
if (hasAscendant(node.resolveParent, compareNodes)) {
return true
}
}
for (const edge of node.edgesIn) {
// TODO Need a test with an infinite loop
if (seen.has(edge)) {
continue
}
seen.add(edge)
if (edge && edge.from && hasAscendant(edge.from, compareNodes, seen)) {
return true
}
}
for (const linkNode of node.linksIn) {
if (hasAscendant(linkNode, compareNodes, seen)) {
return true
}
}
return false
}
const combinators = {
// direct descendant
'>' (prevResults, nextResults) {
return nextResults.filter(node => hasParent(node, prevResults))
},
// any descendant
' ' (prevResults, nextResults) {
return nextResults.filter(node => hasAscendant(node, prevResults))
},
// sibling
'~' (prevResults, nextResults) {
// Return any node in nextResults that is a sibling of (aka shares a
// parent with) a node in prevResults
const parentNodes = new Set() // Parents of everything in prevResults
for (const node of prevResults) {
for (const edge of node.edgesIn) {
// edge.from always exists cause it's from another node's edgesIn
parentNodes.add(edge.from)
}
}
return nextResults.filter(node =>
!prevResults.includes(node) && hasParent(node, [...parentNodes])
)
},
}
// get a list of available versions of a package filtered to respect --before
// NOTE: this runs over each node and should not throw
const getPackageVersions = async (name, opts) => {
let packument
try {
packument = await pacote.packument(name, {
...opts,
fullMetadata: false, // we only need the corgi
})
} catch (err) {
// if the fetch fails, log a warning and pretend there are no versions
log.warn('query', `could not retrieve packument for ${name}: ${err.message}`)
return []
}
// start with a sorted list of all versions (lowest first)
let candidates = Object.keys(packument.versions).sort(semver.compare)
// if the packument has a time property, and the user passed a before flag, then
// we filter this list down to only those versions that existed before the specified date
if (packument.time && opts.before) {
candidates = candidates.filter((version) => {
// this version isn't found in the times at all, drop it
if (!packument.time[version]) {
return false
}
return Date.parse(packument.time[version]) <= opts.before
})
}
return candidates
}
const retrieveNodesFromParsedAst = async (opts) => {
// when we first call this it's the parsed query. all other times it's
// results.currentNode.nestedNode
const rootAstNode = opts.rootAstNode
if (!rootAstNode.nodes) {
return new Set()
}
const results = new Results(opts)
const astNodeQueue = new Set()
// walk is sync, so we have to build up our async functions and then await them later
rootAstNode.walk((nextAstNode) => {
astNodeQueue.add(nextAstNode)
})
for (const nextAstNode of astNodeQueue) {
// This is the only place we reset currentAstNode
results.currentAstNode = nextAstNode
const updateFn = `${results.currentAstNode.type}Type`
if (typeof results[updateFn] !== 'function') {
throw Object.assign(
new Error(`\`${results.currentAstNode.type}\` is not a supported selector.`),
{ code: 'EQUERYNOSELECTOR' }
)
}
await results[updateFn]()
}
return results.collect(rootAstNode)
}
const querySelectorAll = async (targetNode, query, flatOptions) => {
// This never changes ever we just pass it around. But we can't scope it to
// this whole file if we ever want to support concurrent calls to this
// function.
const inventory = [...targetNode.root.inventory.values()]
// res is a Set of items returned for each parsed css ast selector
const res = await retrieveNodesFromParsedAst({
initialItems: inventory,
inventory,
flatOptions,
rootAstNode: parser(query),
targetNode,
})
// returns nodes ordered by realpath
return [...res].sort((a, b) => localeCompare(a.location, b.location))
}
module.exports = querySelectorAll
@@ -0,0 +1,95 @@
// look up the realpath, but cache stats to minimize overhead
// If the parent folder is in the realpath cache, then we just
// lstat the child, since there's no need to do a full realpath
// This is not a separate module, and is much simpler than Node's
// built-in fs.realpath, because we only care about symbolic links,
// so we can handle many fewer edge cases.
const { lstat, readlink } = require('node:fs/promises')
const { resolve, basename, dirname } = require('node:path')
const realpathCached = (path, rpcache, stcache, depth) => {
// just a safety against extremely deep eloops
/* istanbul ignore next */
if (depth > 2000) {
throw eloop(path)
}
path = resolve(path)
if (rpcache.has(path)) {
return Promise.resolve(rpcache.get(path))
}
const dir = dirname(path)
const base = basename(path)
if (base && rpcache.has(dir)) {
return realpathChild(dir, base, rpcache, stcache, depth)
}
// if it's the root, then we know it's real
if (!base) {
rpcache.set(dir, dir)
return Promise.resolve(dir)
}
// the parent, what is that?
// find out, and then come back.
return realpathCached(dir, rpcache, stcache, depth + 1).then(() =>
realpathCached(path, rpcache, stcache, depth + 1))
}
const lstatCached = (path, stcache) => {
if (stcache.has(path)) {
return Promise.resolve(stcache.get(path))
}
const p = lstat(path).then(st => {
stcache.set(path, st)
return st
})
stcache.set(path, p)
return p
}
// This is a slight fib, as it doesn't actually occur during a stat syscall.
// But file systems are giant piles of lies, so whatever.
const eloop = path =>
Object.assign(new Error(
`ELOOP: too many symbolic links encountered, stat '${path}'`), {
errno: -62,
syscall: 'stat',
code: 'ELOOP',
path: path,
})
const realpathChild = (dir, base, rpcache, stcache, depth) => {
const realdir = rpcache.get(dir)
// that unpossible
/* istanbul ignore next */
if (typeof realdir === 'undefined') {
throw new Error('in realpathChild without parent being in realpath cache')
}
const realish = resolve(realdir, base)
return lstatCached(realish, stcache).then(st => {
if (!st.isSymbolicLink()) {
rpcache.set(resolve(dir, base), realish)
return realish
}
return readlink(realish).then(target => {
const resolved = resolve(realdir, target)
if (realish === resolved) {
throw eloop(realish)
}
return realpathCached(resolved, rpcache, stcache, depth + 1)
}).then(real => {
rpcache.set(resolve(dir, base), real)
return real
})
})
}
module.exports = realpathCached
@@ -0,0 +1,3 @@
const { relative } = require('node:path')
const relpath = (from, to) => relative(from, to).replace(/\\/g, '/')
module.exports = relpath
@@ -0,0 +1,11 @@
// Sometimes we need to actually do a walk from the root, because you can
// have a cycle of deps that all depend on each other, but no path from root.
// Also, since the ideal tree is loaded from the shrinkwrap, it had extraneous
// flags set false that might now be actually extraneous, and dev/optional
// flags that are also now incorrect. This method sets all flags to true, so
// we can find the set that is actually extraneous.
module.exports = tree => {
for (const node of tree.inventory.values()) {
node.resetDepFlags()
}
}
@@ -0,0 +1,19 @@
const crypto = require('node:crypto')
const { dirname, basename, resolve } = require('node:path')
// use sha1 because it's faster, and collisions extremely unlikely anyway
const pathSafeHash = s =>
crypto.createHash('sha1')
.update(s)
.digest('base64')
.replace(/[^a-zA-Z0-9]+/g, '')
.slice(0, 8)
const retirePath = from => {
const d = dirname(from)
const b = basename(from)
const hash = pathSafeHash(from)
return resolve(d, `.${b}-${hash}`)
}
module.exports = retirePath
@@ -0,0 +1,340 @@
const npa = require('npm-package-arg')
const semver = require('semver')
const versionFromTgz = require('./version-from-tgz.js')
// Identity matcher for the allowScripts policy.
//
// Returns:
// - true: at least one allow entry matches and no deny entry matches
// - false: at least one deny entry matches (deny wins on conflict)
// - null: no entry matches (unreviewed)
//
// `policy` is a flat object of `spec-key -> boolean`, where spec-key is
// anything `npm-package-arg` can parse. `node` is an arborist Node.
//
// Identity rules (see RFC npm/rfcs#868):
// - registry deps match by the name+version parsed from the lockfile's
// resolved URL, NOT by `node.packageName` / `node.version`. Those two
// getters return `node.package.name` / `node.package.version`, which
// come from the tarball's own package.json and are therefore
// attacker-controlled. A package can publish a tarball claiming any
// name; the only trusted name is the one baked into the registry URL.
// - tarball / file / link / remote: exact match on node.resolved
// - git: match on hosted.ssh() plus a short-SHA prefix of the
// resolved committish
const isScriptAllowed = (node, policy) => {
// Bundled dependencies cannot be allowlisted in Phase 1. The RFC defers
// allowlisting them to a follow-up RFC because matching by name@version
// from the bundled tarball would reintroduce manifest confusion (a
// bundled tarball can claim any name and version). Returning null here
// marks bundled deps as unreviewed regardless of any policy entries, so
// their install scripts surface in the Phase 1 advisory warning and
// (eventually) get blocked at the install-time gate.
if (node.inBundle) {
return null
}
if (!policy || typeof policy !== 'object') {
return null
}
let anyAllow = false
let anyDeny = false
for (const [key, value] of Object.entries(policy)) {
if (!matches(node, key)) {
continue
}
if (value === false) {
anyDeny = true
continue
}
/* istanbul ignore else: policy values are strictly true/false;
defensive guard against unexpected coercions. */
if (value === true) {
anyAllow = true
}
}
if (anyDeny) {
return false
}
if (anyAllow) {
return true
}
return null
}
const matches = (node, key) => {
let parsed
try {
parsed = npa(key)
} catch {
return false
}
switch (parsed.type) {
case 'tag':
case 'range':
case 'version':
return matchRegistry(node, parsed)
case 'git':
return matchGit(node, parsed)
case 'file':
case 'directory':
return matchFileOrDir(node, parsed)
case 'remote':
return matchRemote(node, parsed)
case 'alias':
// Disallowed: aliases as policy keys do not match anything.
// The user has to address the real package name.
return false
/* istanbul ignore next: switch above covers every npa type we expect;
defensive fallback for future npa types. */
default:
return false
}
}
const matchRegistry = (node, parsed) => {
// If this node is not a registry dep, refuse the match. A registry-style
// key (`pkg`, `pkg@1`, `pkg@1 || 2`) must not match a tarball or git node
// even if their names happen to coincide.
if (!isRegistryNode(node)) {
return false
}
// Derive the trusted name+version from the lockfile's resolved URL.
// Never use `node.packageName` / `node.version` here: those read from
// the tarball's own package.json and can be forged by a malicious
// publisher to bypass an allowScripts entry.
const trusted = getTrustedRegistryIdentity(node)
if (!trusted || trusted.name !== parsed.name) {
return false
}
// `tag` covers `pkg@latest`. Rejected up front by validatePolicy in
// resolve-allow-scripts.js because tags look like a pin but can't be
// verified at install time. Defense-in-depth: if one slips through
// (e.g. arborist invoked directly without the resolver), don't match.
if (parsed.type === 'tag') {
/* istanbul ignore next: validatePolicy filters this; defensive */
return false
}
// `range` includes `pkg@^1`, `pkg@1 || 2`, `pkg@*`, `pkg@>=0`, and bare
// names like `pkg` (npa parses these as range with fetchSpec='*'). The
// RFC permits bare names (name-only allow) and exact versions joined by
// `||`; ranges like ^/~/>=/< are rejected because they would silently
// allow versions the user has never reviewed.
if (parsed.type === 'range') {
// Bare name or `pkg@*`: treat as name-only allow.
if (parsed.fetchSpec === '*' || parsed.rawSpec === '' || parsed.rawSpec === '*') {
return true
}
if (!trusted.version || !isExactVersionDisjunction(parsed.fetchSpec)) {
return false
}
return semver.satisfies(trusted.version, parsed.fetchSpec, { loose: true })
}
// `version` is an exact pin like `pkg@1.2.3`.
/* istanbul ignore else: parsed.type at this point is always 'version';
the istanbul-ignored fallback below handles the impossible case. */
if (parsed.type === 'version') {
return trusted.version === parsed.fetchSpec
}
/* istanbul ignore next: parsed.type is constrained to tag/range/version
by the caller; this final fallback is defensive. */
return false
}
// Derive a registry node's trusted name+version.
//
// Preferred source: the lockfile's resolved URL parsed via
// versionFromTgz. arborist records the URL when it first adds the dep,
// before any tarball is unpacked, so the URL cannot be forged by the
// package's own package.json.
//
// Fallback for lockfiles produced with omit-lockfile-registry-resolved
// (where the URL is absent): take the dep name from an incoming
// dependency edge. The edge's spec was written by the consumer (or by an
// upstream package.json), not by the installed tarball. For aliases like
// `"trusted": "npm:naughty@1.0.0"`, the underlying registered package
// name is parsed out of the alias `subSpec`. The install location
// (`node_modules/trusted`) is deliberately not consulted because for
// aliases it carries only the alias name, which would let a malicious
// publisher bypass an allowScripts entry written for the real package.
//
// Version is left null in the fallback case because the only remaining
// source for it (`node.version`) reads from the tarball.
//
// Returns `{ name, version }` or `null` if no trusted identity exists.
const getTrustedRegistryIdentity = (node) => {
if (node.resolved && typeof node.resolved === 'string') {
const parsed = versionFromTgz('', node.resolved)
/* istanbul ignore else: versionFromTgz returns either a complete
{ name, version } or null; partial objects are not produced. */
if (parsed && parsed.name && parsed.version) {
return parsed
}
}
const name = nameFromEdges(node)
if (name) {
return { name, version: null }
}
return null
}
const nameFromEdges = (node) => {
if (!node.edgesIn || typeof node.edgesIn[Symbol.iterator] !== 'function') {
return null
}
for (const edge of node.edgesIn) {
let parsed
try {
parsed = npa.resolve(edge.name, edge.spec)
} catch {
continue
}
// Aliases: trust the underlying registered package, not the alias.
if (parsed.type === 'alias' && parsed.subSpec && parsed.subSpec.registry) {
return parsed.subSpec.name
}
// Non-aliased registry edge: the edge name is the package name as
// written by the consumer / upstream, which is trusted (it is not
// read from the installed tarball).
if (parsed.registry) {
return parsed.name
}
}
return null
}
// True if `rangeSpec` is one or more exact versions joined by `||`. Anything
// containing comparator operators (^, ~, >=, <, *) returns false.
const isExactVersionDisjunction = (rangeSpec) => {
/* istanbul ignore next: caller always passes parsed.fetchSpec, which
npa guarantees to be a non-empty string for range specs. */
if (typeof rangeSpec !== 'string' || rangeSpec.trim() === '') {
return false
}
const parts = rangeSpec.split('||').map(p => p.trim())
/* istanbul ignore next: String.prototype.split always returns at least
one element; defensive guard only. */
if (parts.length === 0) {
return false
}
return parts.every(p => p !== '' && semver.valid(p) !== null)
}
const matchGit = (node, parsed) => {
if (!node.resolved || !node.resolved.startsWith('git')) {
return false
}
let nodeParsed
try {
nodeParsed = npa(node.resolved)
} catch {
/* istanbul ignore next: npa parsing a git URL we already validated
starts with `git` should not throw; defensive guard only. */
return false
}
// Compare the host/repo. Both sides should resolve to the same canonical
// ssh URL.
const noCommittish = { noCommittish: true }
const keyHost = parsed.hosted?.ssh(noCommittish)
const nodeHost = nodeParsed.hosted?.ssh(noCommittish)
if (keyHost && nodeHost) {
if (keyHost !== nodeHost) {
return false
}
} else if (parsed.fetchSpec && nodeParsed.fetchSpec) {
// Non-hosted git URLs: fall back to fetch spec.
if (parsed.fetchSpec !== nodeParsed.fetchSpec) {
return false
}
} else {
return false
}
// If the policy key has no committish, name-only match.
const keyCommittish = parsed.gitCommittish || parsed.hosted?.committish
if (!keyCommittish) {
return true
}
// Match the resolved full SHA against the key's committish. Users
// typically write short SHAs in the policy; the lockfile stores 40-char
// SHAs. Direction matters: the lockfile's full SHA must START WITH the
// key's short SHA, never the reverse. A longer key matching a shorter
// resolved committish would let a malformed lockfile or a divergent
// resolver allow scripts the user never approved.
const nodeCommittish = nodeParsed.gitCommittish || nodeParsed.hosted?.committish || ''
if (!nodeCommittish) {
return false
}
return nodeCommittish.startsWith(keyCommittish)
}
const matchFileOrDir = (node, parsed) => {
if (!node.resolved) {
return false
}
return node.resolved === parsed.saveSpec || node.resolved === parsed.fetchSpec
}
const matchRemote = (node, parsed) => {
if (!node.resolved) {
return false
}
return node.resolved === parsed.fetchSpec || node.resolved === parsed.saveSpec
}
const isRegistryNode = (node) => {
// Prefer arborist's edge-based check when available (real Node objects).
// It inspects the incoming edges' specs and only returns true if every
// edge resolves to a registry spec, which is much harder to spoof than
// the URL.
if (typeof node.isRegistryDependency === 'boolean') {
return node.isRegistryDependency
}
// Fall back to URL parsing for nodes without the arborist getter
// (e.g. test fixtures, lockfiles with omit-lockfile-registry-resolved).
// Treat the node as a registry dep when:
// - resolved is missing entirely (omitLockfileRegistryResolved),
// - resolved is an https/http URL pointing at a registry tarball, or
// - resolved is undefined and the node has a version (defensive).
if (!node.resolved) {
return !!node.version
}
// Registry tarballs live at `<host>/<pkg-name>/-/<pkg-name>-<version>.tgz`.
// Require a path segment before `/-/` so an attacker can't lift a
// registry-style allow entry to a hostile URL like
// `https://evil.com/-/trusted-1.0.0.tgz`.
return /^https?:\/\/[^/]+\/.+\/-\/[^/]+-\d/.test(node.resolved)
}
// Trusted display identity for human-facing output (`npm install`
// advisory, `npm approve-scripts --allow-scripts-pending`). Same idea as
// getTrustedRegistryIdentity, but for DISPLAY only — version falls back
// to node.version when the URL doesn't carry one. Must never be used
// for policy matching.
const trustedDisplay = (node) => {
const trusted = getTrustedRegistryIdentity(node)
/* istanbul ignore next: defensive fallbacks for nodes without name/version */
return {
name: (trusted && trusted.name) || node.name || null,
version: (trusted && trusted.version) || node.version || null,
}
}
module.exports = isScriptAllowed
module.exports.isScriptAllowed = isScriptAllowed
module.exports.isExactVersionDisjunction = isExactVersionDisjunction
module.exports.getTrustedRegistryIdentity = getTrustedRegistryIdentity
module.exports.trustedDisplay = trustedDisplay
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,74 @@
const signals = require('./signals.js')
// for testing, expose the process being used
module.exports = Object.assign(fn => setup(fn), { process })
// do all of this in a setup function so that we can call it
// multiple times for multiple reifies that might be going on.
// Otherwise, Arborist.reify() is a global action, which is a
// new constraint we'd be adding with this behavior.
const setup = fn => {
const { process } = module.exports
const sigListeners = { loaded: false }
const unload = () => {
if (!sigListeners.loaded) {
return
}
for (const sig of signals) {
try {
process.removeListener(sig, sigListeners[sig])
} catch {
// ignore errors
}
}
process.removeListener('beforeExit', onBeforeExit)
sigListeners.loaded = false
}
const onBeforeExit = () => {
// this trick ensures that we exit with the same signal we caught
// Ie, if you press ^C and npm gets a SIGINT, we'll do the rollback
// and then exit with a SIGINT signal once we've removed the handler.
// The timeout is there because signals are asynchronous, so we need
// the process to NOT exit on its own, which means we have to have
// something keeping the event loop looping. Hence this hack.
unload()
process.kill(process.pid, signalReceived)
setTimeout(() => {}, 500)
}
let signalReceived = null
const listener = (sig, fn) => () => {
signalReceived = sig
// if we exit normally, but caught a signal which would have been fatal,
// then re-send it once we're done with whatever cleanup we have to do.
unload()
if (process.listeners(sig).length < 1) {
process.once('beforeExit', onBeforeExit)
}
fn({ signal: sig })
}
// do the actual loading here
for (const sig of signals) {
sigListeners[sig] = listener(sig, fn)
const max = process.getMaxListeners()
try {
// if we call this a bunch of times, avoid triggering the warning
const { length } = process.listeners(sig)
if (length >= max) {
process.setMaxListeners(length + 1)
}
process.on(sig, sigListeners[sig])
} catch {
// ignore errors
}
}
sigListeners.loaded = true
return unload
}
@@ -0,0 +1,58 @@
// copied from signal-exit
// This is not the set of all possible signals.
//
// It IS, however, the set of all signals that trigger
// an exit on either Linux or BSD systems. Linux is a
// superset of the signal names supported on BSD, and
// the unknown signals just fail to register, so we can
// catch that easily enough.
//
// Don't bother with SIGKILL. It's uncatchable, which
// means that we can't fire any callbacks anyway.
//
// If a user does happen to register a handler on a non-
// fatal signal like SIGWINCH or something, and then
// exit, it'll end up firing `process.emit('exit')`, so
// the handler will be fired anyway.
//
// SIGBUS, SIGFPE, SIGSEGV and SIGILL, when not raised
// artificially, inherently leave the process in a
// state from which it is not safe to try and enter JS
// listeners.
const platform = global.__ARBORIST_FAKE_PLATFORM__ || process.platform
module.exports = [
'SIGABRT',
'SIGALRM',
'SIGHUP',
'SIGINT',
'SIGTERM',
]
if (platform !== 'win32') {
module.exports.push(
'SIGVTALRM',
'SIGXCPU',
'SIGXFSZ',
'SIGUSR2',
'SIGTRAP',
'SIGSYS',
'SIGQUIT',
'SIGIOT'
// should detect profiler and enable/disable accordingly.
// see #21
// 'SIGPROF'
)
}
if (platform === 'linux') {
module.exports.push(
'SIGIO',
'SIGPOLL',
'SIGPWR',
'SIGSTKFLT',
'SIGUNUSED'
)
}
@@ -0,0 +1,34 @@
const npa = require('npm-package-arg')
// extracted from npm v6 lib/install/realize-shrinkwrap-specifier.js
const specFromLock = (name, lock, where) => {
try {
if (lock.version) {
const spec = npa.resolve(name, lock.version, where)
if (lock.integrity || spec.type === 'git') {
return spec
}
}
if (lock.from) {
// legacy metadata includes "from", but not integrity
const spec = npa.resolve(name, lock.from, where)
if (spec.registry && lock.version) {
return npa.resolve(name, lock.version, where)
} else if (!lock.resolved) {
return spec
}
}
if (lock.resolved) {
return npa.resolve(name, lock.resolved, where)
}
} catch {
// ignore errors
}
try {
return npa.resolve(name, lock.version, where)
} catch {
return {}
}
}
module.exports = specFromLock
@@ -0,0 +1,86 @@
const proggy = require('proggy')
module.exports = cls => class Tracker extends cls {
#progress = new Map()
#createTracker (key, name) {
const tracker = new proggy.Tracker(name ?? key)
tracker.on('done', () => this.#progress.delete(key))
this.#progress.set(key, tracker)
}
addTracker (section, subsection = null, key = null) {
if (section === null || section === undefined) {
this.#onError(`Tracker can't be null or undefined`)
}
if (key === null) {
key = subsection
}
const hasTracker = this.#progress.has(section)
const hasSubtracker = this.#progress.has(`${section}:${key}`)
if (hasTracker && subsection === null) {
// 0. existing tracker, no subsection
this.#onError(`Tracker "${section}" already exists`)
} else if (!hasTracker && subsection === null) {
// 1. no existing tracker, no subsection
// Create a new progress tracker
this.#createTracker(section)
} else if (!hasTracker && subsection !== null) {
// 2. no parent tracker and subsection
this.#onError(`Parent tracker "${section}" does not exist`)
} else if (!hasTracker || !hasSubtracker) {
// 3. existing parent tracker, no subsection tracker
// Create a new subtracker and update parents
const parentTracker = this.#progress.get(section)
parentTracker.update(parentTracker.value, parentTracker.total + 1)
this.#createTracker(`${section}:${key}`, `${section}:${subsection}`)
}
// 4. existing parent tracker, existing subsection tracker
// skip it
}
finishTracker (section, subsection = null, key = null) {
if (section === null || section === undefined) {
this.#onError(`Tracker can't be null or undefined`)
}
if (key === null) {
key = subsection
}
const hasTracker = this.#progress.has(section)
const hasSubtracker = this.#progress.has(`${section}:${key}`)
// 0. parent tracker exists, no subsection
// Finish parent tracker and remove from this.#progress
if (hasTracker && subsection === null) {
// check if parent tracker does
// not have any remaining children
const keys = this.#progress.keys()
for (const key of keys) {
if (key.match(new RegExp(section + ':'))) {
this.finishTracker(section, key)
}
}
// remove parent tracker
this.#progress.get(section).finish()
} else if (!hasTracker && subsection === null) {
// 1. no existing parent tracker, no subsection
this.#onError(`Tracker "${section}" does not exist`)
} else if (!hasTracker || hasSubtracker) {
// 2. subtracker exists
// Finish subtracker and remove from this.#progress
const parentTracker = this.#progress.get(section)
parentTracker.update(parentTracker.value + 1)
this.#progress.get(`${section}:${key}`).finish()
}
// 3. existing parent tracker, no subsection
}
#onError (msg) {
throw new Error(msg)
}
}
@@ -0,0 +1,155 @@
const debug = require('./debug.js')
const checkTree = (tree, checkUnreachable = true) => {
const log = [['START TREE CHECK', tree.path]]
// this can only happen in tests where we have a "tree" object
// that isn't actually a tree.
if (!tree.root || !tree.root.inventory) {
return tree
}
const { inventory } = tree.root
const seen = new Set()
const check = (node, via = tree, viaType = 'self') => {
log.push([
'CHECK',
node && node.location,
via && via.location,
viaType,
'seen=' + seen.has(node),
'promise=' + !!(node && node.then),
'root=' + !!(node && node.isRoot),
])
if (!node || seen.has(node) || node.then) {
return
}
seen.add(node)
if (node.isRoot && node !== tree.root) {
throw Object.assign(new Error('double root'), {
node: node.path,
realpath: node.realpath,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
log,
})
}
if (node.root !== tree.root) {
throw Object.assign(new Error('node from other root in tree'), {
node: node.path,
realpath: node.realpath,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
otherRoot: node.root && node.root.path,
log,
})
}
if (!node.isRoot && node.inventory.size !== 0) {
throw Object.assign(new Error('non-root has non-zero inventory'), {
node: node.path,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
inventory: [...node.inventory.values()].map(node =>
[node.path, node.location]),
log,
})
}
if (!node.isRoot && !inventory.has(node) && !node.dummy) {
throw Object.assign(new Error('not in inventory'), {
node: node.path,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
log,
})
}
const devEdges = [...node.edgesOut.values()].filter(e => e.dev)
if (!node.isTop && devEdges.length) {
throw Object.assign(new Error('dev edges on non-top node'), {
node: node.path,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
devEdges: devEdges.map(e => [e.type, e.name, e.spec, e.error]),
log,
})
}
if (node.path === tree.root.path && node !== tree.root && !tree.root.isLink) {
throw Object.assign(new Error('node with same path as root'), {
node: node.path,
tree: tree.path,
root: tree.root.path,
via: via.path,
viaType,
log,
})
}
if (!node.isLink && node.path !== node.realpath) {
throw Object.assign(new Error('non-link with mismatched path/realpath'), {
node: node.path,
tree: tree.path,
realpath: node.realpath,
root: tree.root.path,
via: via.path,
viaType,
log,
})
}
const { parent, fsParent, target } = node
check(parent, node, 'parent')
check(fsParent, node, 'fsParent')
check(target, node, 'target')
log.push(['CHILDREN', node.location, ...node.children.keys()])
for (const kid of node.children.values()) {
check(kid, node, 'children')
}
for (const kid of node.fsChildren) {
check(kid, node, 'fsChildren')
}
for (const link of node.linksIn) {
check(link, node, 'linksIn')
}
for (const top of node.tops) {
check(top, node, 'tops')
}
log.push(['DONE', node.location])
}
check(tree)
if (checkUnreachable) {
for (const node of inventory.values()) {
if (!seen.has(node) && node !== tree.root) {
throw Object.assign(new Error('unreachable in inventory'), {
node: node.path,
realpath: node.realpath,
location: node.location,
root: tree.root.path,
tree: tree.path,
log,
})
}
}
}
return tree
}
// should only ever run this check in debug mode
module.exports = tree => tree
debug(() => module.exports = checkTree)
@@ -0,0 +1,47 @@
const semver = require('semver')
const { basename } = require('node:path')
const { URL } = require('node:url')
module.exports = (name, tgz) => {
const base = basename(tgz)
if (!base.endsWith('.tgz')) {
return null
}
if (tgz.startsWith('http:/') || tgz.startsWith('https:/')) {
const u = new URL(tgz)
// registry url? check for most likely pattern.
// either /@foo/bar/-/bar-1.2.3.tgz or
// /foo/-/foo-1.2.3.tgz, and fall through to
// basename checking. Note that registries can
// be mounted below the root url, so /a/b/-/x/y/foo/-/foo-1.2.3.tgz
// is a potential option.
const tfsplit = u.pathname.slice(1).split('/-/')
if (tfsplit.length > 1) {
const afterTF = tfsplit.pop()
if (afterTF === base) {
const pre = tfsplit.pop()
const preSplit = pre.split(/\/|%2f/i)
const project = preSplit.pop()
const scope = preSplit.pop()
return versionFromBaseScopeName(base, scope, project)
}
}
}
const split = name.split(/\/|%2f/i)
const project = split.pop()
const scope = split.pop()
return versionFromBaseScopeName(base, scope, project)
}
const versionFromBaseScopeName = (base, scope, name) => {
if (!base.startsWith(name + '-')) {
return null
}
const parsed = semver.parse(base.substring(name.length + 1, base.length - 4))
return parsed ? {
name: scope && scope.charAt(0) === '@' ? `${scope}/${name}` : name,
version: parsed.version,
} : null
}
@@ -0,0 +1,214 @@
// An object representing a vulnerability either as the result of an
// advisory or due to the package in question depending exclusively on
// vulnerable versions of a dep.
//
// - name: package name
// - range: Set of vulnerable versions
// - nodes: Set of nodes affected
// - effects: Set of vulns triggered by this one
// - advisories: Set of advisories (including metavulns) causing this vuln.
// All of the entries in via are vulnerability objects returned by
// @npmcli/metavuln-calculator
// - via: dependency vulns which cause this one
const { satisfies, simplifyRange } = require('semver')
const semverOpt = { loose: true, includePrerelease: true }
const localeCompare = require('@isaacs/string-locale-compare')('en')
const npa = require('npm-package-arg')
const severities = new Map([
['info', 0], [0, 'info'],
['low', 1], [1, 'low'],
['moderate', 2], [2, 'moderate'],
['high', 3], [3, 'high'],
['critical', 4], [4, 'critical'],
[null, -1], [-1, null],
])
class Vuln {
#range = null
#simpleRange = null
// assume a fix is available unless it hits a top node
// that locks it in place, setting this false or {isSemVerMajor, version}.
#fixAvailable = true
constructor ({ name, advisory }) {
this.name = name
this.via = new Set()
this.advisories = new Set()
this.severity = null
this.effects = new Set()
this.topNodes = new Set()
this.nodes = new Set()
this.addAdvisory(advisory)
this.packument = advisory.packument
this.versions = advisory.versions
}
get fixAvailable () {
return this.#fixAvailable
}
set fixAvailable (f) {
this.#fixAvailable = f
// if there's a fix available for this at the top level, it means that
// it will also fix the vulns that led to it being there. to get there,
// we set the vias to the most "strict" of fix availables.
// - false: no fix is available
// - {name, version, isSemVerMajor} fix requires -f, is semver major
// - {name, version} fix requires -f, not semver major
// - true: fix does not require -f
// TODO: duped entries may require different fixes but the current
// structure does not support this, so the case were a top level fix
// corrects a duped entry may mean you have to run fix more than once
for (const v of this.via) {
// don't blow up on loops
if (v.fixAvailable === f) {
continue
}
if (f === false) {
v.fixAvailable = f
} else if (v.fixAvailable === true) {
v.fixAvailable = f
} else if (typeof f === 'object' && (
typeof v.fixAvailable !== 'object' || !v.fixAvailable.isSemVerMajor)) {
v.fixAvailable = f
}
}
}
get isDirect () {
for (const node of this.nodes.values()) {
for (const edge of node.edgesIn) {
if (edge.from.isProjectRoot || edge.from.isWorkspace) {
return true
}
}
}
return false
}
testSpec (spec) {
const specObj = npa(spec)
if (!specObj.registry) {
return true
}
if (specObj.subSpec) {
spec = specObj.subSpec.rawSpec
}
for (const v of this.versions) {
if (satisfies(v, spec) && !satisfies(v, this.range, semverOpt)) {
return false
}
}
return true
}
toJSON () {
return {
name: this.name,
severity: this.severity,
isDirect: this.isDirect,
// just loop over the advisories, since via is only Vuln objects,
// and calculated advisories have all the info we need
via: [...this.advisories].map(v => v.type === 'metavuln' ? v.dependency : {
...v,
versions: undefined,
vulnerableVersions: undefined,
id: undefined,
}).sort((a, b) =>
localeCompare(String(a.source || a), String(b.source || b))),
effects: [...this.effects].map(v => v.name).sort(localeCompare),
range: this.simpleRange,
nodes: [...this.nodes].map(n => n.location).sort(localeCompare),
fixAvailable: this.#fixAvailable,
}
}
addVia (v) {
this.via.add(v)
v.effects.add(this)
// call the setter since we might add vias _after_ setting fixAvailable
this.fixAvailable = this.fixAvailable
}
deleteVia (v) {
this.via.delete(v)
v.effects.delete(this)
}
deleteAdvisory (advisory) {
this.advisories.delete(advisory)
// make sure we have the max severity of all the vulns causing this one
this.severity = null
this.#range = null
this.#simpleRange = null
// refresh severity
for (const advisory of this.advisories) {
this.addAdvisory(advisory)
}
// remove any effects that are no longer relevant
const vias = new Set([...this.advisories].map(a => a.dependency))
for (const via of this.via) {
if (!vias.has(via.name)) {
this.deleteVia(via)
}
}
}
addAdvisory (advisory) {
this.advisories.add(advisory)
const sev = severities.get(advisory.severity)
this.#range = null
this.#simpleRange = null
if (sev > severities.get(this.severity)) {
this.severity = advisory.severity
}
}
get range () {
if (!this.#range) {
this.#range = [...this.advisories].map(v => v.range).join(' || ')
}
return this.#range
}
get simpleRange () {
if (this.#simpleRange && this.#simpleRange === this.#range) {
return this.#simpleRange
}
const versions = [...this.advisories][0].versions
const range = this.range
this.#simpleRange = simplifyRange(versions, range, semverOpt)
this.#range = this.#simpleRange
return this.#simpleRange
}
isVulnerable (node) {
if (this.nodes.has(node)) {
return true
}
const { version } = node.package
if (!version) {
return false
}
for (const v of this.advisories) {
if (v.testVersion(version)) {
this.nodes.add(node)
return true
}
}
return false
}
}
module.exports = Vuln
@@ -0,0 +1,377 @@
// parse a yarn lock file
// basic format
//
// <request spec>[, <request spec> ...]:
// <key> <value>
// <subkey>:
// <key> <value>
//
// Assume that any key or value might be quoted, though that's only done
// in practice if certain chars are in the string. When writing back, we follow
// Yarn's rules for quoting, to cause minimal friction.
//
// The data format would support nested objects, but at this time, it
// appears that yarn does not use that for anything, so in the interest
// of a simpler parser algorithm, this implementation only supports a
// single layer of sub objects.
//
// This doesn't deterministically define the shape of the tree, and so
// cannot be used (on its own) for Arborist.loadVirtual.
// But it can give us resolved, integrity, and version, which is useful
// for Arborist.loadActual and for building the ideal tree.
//
// At the very least, when a yarn.lock file is present, we update it
// along the way, and save it back in Shrinkwrap.save()
//
// NIHing this rather than using @yarnpkg/lockfile because that module
// is an impenetrable 10kloc of webpack flow output, which is overkill
// for something relatively simple and tailored to Arborist's use case.
const localeCompare = require('@isaacs/string-locale-compare')('en')
const consistentResolve = require('./consistent-resolve.js')
const { dirname } = require('node:path')
const { breadth } = require('treeverse')
// Sort Yarn entries respecting the yarn.lock sort order
const yarnEntryPriorities = {
name: 1,
version: 2,
uid: 3,
resolved: 4,
integrity: 5,
registry: 6,
dependencies: 7,
}
const priorityThenLocaleCompare = (a, b) => {
if (!yarnEntryPriorities[a] && !yarnEntryPriorities[b]) {
return localeCompare(a, b)
}
/* istanbul ignore next */
return (yarnEntryPriorities[a] || 100) > (yarnEntryPriorities[b] || 100) ? 1 : -1
}
const quoteIfNeeded = val => {
if (
typeof val === 'boolean' ||
typeof val === 'number' ||
val.startsWith('true') ||
val.startsWith('false') ||
/[:\s\n\\",[\]]/g.test(val) ||
!/^[a-zA-Z]/g.test(val)
) {
return JSON.stringify(val)
}
return val
}
// sort a key/value object into a string of JSON stringified keys and vals
const sortKV = obj => Object.keys(obj)
.sort(localeCompare)
.map(k => ` ${quoteIfNeeded(k)} ${quoteIfNeeded(obj[k])}`)
.join('\n')
// for checking against previous entries
const match = (p, n) =>
p.integrity && n.integrity ? p.integrity === n.integrity
: p.resolved && n.resolved ? p.resolved === n.resolved
: p.version && n.version ? p.version === n.version
: true
const prefix =
`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
`
const nullSymbol = Symbol('null')
class YarnLock {
static parse (data) {
return new YarnLock().parse(data)
}
static fromTree (tree) {
return new YarnLock().fromTree(tree)
}
constructor () {
this.entries = null
this.endCurrent()
}
endCurrent () {
this.current = null
this.subkey = nullSymbol
}
parse (data) {
const ENTRY_START = /^[^\s].*:$/
const SUBKEY = /^ {2}[^\s]+:$/
const SUBVAL = /^ {4}[^\s]+ .+$/
const METADATA = /^ {2}[^\s]+ .+$/
this.entries = new Map()
this.current = null
const linere = /([^\r\n]*)\r?\n/gm
let match
let lineNum = 0
if (!/\n$/.test(data)) {
data += '\n'
}
while (match = linere.exec(data)) {
const line = match[1]
lineNum++
if (line.charAt(0) === '#') {
continue
}
if (line === '') {
this.endCurrent()
continue
}
if (ENTRY_START.test(line)) {
this.endCurrent()
const specs = this.splitQuoted(line.slice(0, -1), /, */)
this.current = new YarnLockEntry(specs)
specs.forEach(spec => this.entries.set(spec, this.current))
continue
}
if (SUBKEY.test(line)) {
this.subkey = line.slice(2, -1)
this.current[this.subkey] = {}
continue
}
if (SUBVAL.test(line) && this.current && this.current[this.subkey]) {
const subval = this.splitQuoted(line.trimLeft(), ' ')
if (subval.length === 2) {
this.current[this.subkey][subval[0]] = subval[1]
continue
}
}
// any other metadata
if (METADATA.test(line) && this.current) {
const metadata = this.splitQuoted(line.trimLeft(), ' ')
if (metadata.length === 2) {
// strip off the legacy shasum hashes
if (metadata[0] === 'resolved') {
metadata[1] = metadata[1].replace(/#.*/, '')
}
this.current[metadata[0]] = metadata[1]
continue
}
}
throw Object.assign(new Error('invalid or corrupted yarn.lock file'), {
position: match.index,
content: match[0],
line: lineNum,
})
}
this.endCurrent()
return this
}
splitQuoted (str, delim) {
// a,"b,c",d"e,f => ['a','"b','c"','d"e','f'] => ['a','b,c','d"e','f']
const split = str.split(delim)
const out = []
let o = 0
for (let i = 0; i < split.length; i++) {
const chunk = split[i]
if (/^".*"$/.test(chunk)) {
out[o++] = chunk.trim().slice(1, -1)
} else if (/^"/.test(chunk)) {
let collect = chunk.trimLeft().slice(1)
while (++i < split.length) {
const n = split[i]
// something that is not a slash, followed by an even number
// of slashes then a " then end => ending on an unescaped "
if (/[^\\](\\\\)*"$/.test(n)) {
collect += n.trimRight().slice(0, -1)
break
} else {
collect += n
}
}
out[o++] = collect
} else {
out[o++] = chunk.trim()
}
}
return out
}
toString () {
return prefix + [...new Set([...this.entries.values()])]
.map(e => e.toString())
.sort((a, b) => localeCompare(a.replace(/"/g, ''), b.replace(/"/g, ''))).join('\n\n') + '\n'
}
fromTree (tree) {
this.entries = new Map()
// walk the tree in a deterministic order, breadth-first, alphabetical
breadth({
tree,
visit: node => this.addEntryFromNode(node),
getChildren: node => [...node.children.values(), ...node.fsChildren]
.sort((a, b) => a.depth - b.depth || localeCompare(a.name, b.name)),
})
return this
}
addEntryFromNode (node) {
const specs = [...node.edgesIn]
.map(e => `${node.name}@${e.spec}`)
.sort(localeCompare)
// Note:
// yarn will do excessive duplication in a case like this:
// root -> (x@1.x, y@1.x, z@1.x)
// y@1.x -> (x@1.1, z@2.x)
// z@1.x -> ()
// z@2.x -> (x@1.x)
//
// where x@1.2 exists, because the "x@1.x" spec will *always* resolve
// to x@1.2, which doesn't work for y's dep on x@1.1, so you'll get this:
//
// root
// +-- x@1.2.0
// +-- y
// | +-- x@1.1.0
// | +-- z@2
// | +-- x@1.2.0
// +-- z@1
//
// instead of this more deduped tree that arborist builds by default:
//
// root
// +-- x@1.2.0 (dep is x@1.x, from root)
// +-- y
// | +-- x@1.1.0
// | +-- z@2 (dep on x@1.x deduped to x@1.1.0 under y)
// +-- z@1
//
// In order to not create an invalid yarn.lock file with conflicting
// entries, AND not tell yarn to create an invalid tree, we need to
// ignore the x@1.x spec coming from z, since it's already in the entries.
//
// So, if the integrity and resolved don't match a previous entry, skip it.
// We call this method on shallower nodes first, so this is fine.
const n = this.entryDataFromNode(node)
let priorEntry = null
const newSpecs = []
for (const s of specs) {
const prev = this.entries.get(s)
// no previous entry for this spec at all, so it's new
if (!prev) {
// if we saw a match already, then assign this spec to it as well
if (priorEntry) {
priorEntry.addSpec(s)
} else {
newSpecs.push(s)
}
continue
}
const m = match(prev, n)
// there was a prior entry, but a different thing. skip this one
if (!m) {
continue
}
// previous matches, but first time seeing it, so already has this spec.
// go ahead and add all the previously unseen specs, though
if (!priorEntry) {
priorEntry = prev
for (const s of newSpecs) {
priorEntry.addSpec(s)
this.entries.set(s, priorEntry)
}
newSpecs.length = 0
continue
}
// have a prior entry matching n, and matching the prev we just saw
// add the spec to it
priorEntry.addSpec(s)
this.entries.set(s, priorEntry)
}
// if we never found a matching prior, then this is a whole new thing
if (!priorEntry) {
const entry = Object.assign(new YarnLockEntry(newSpecs), n)
for (const s of newSpecs) {
this.entries.set(s, entry)
}
} else {
// pick up any new info that we got for this node, so that we can
// decorate with integrity/resolved/etc.
Object.assign(priorEntry, n)
}
}
entryDataFromNode (node) {
const n = {}
if (node.package.dependencies) {
n.dependencies = node.package.dependencies
}
if (node.package.optionalDependencies) {
n.optionalDependencies = node.package.optionalDependencies
}
if (node.version) {
n.version = node.version
}
if (node.resolved) {
n.resolved = consistentResolve(
node.resolved,
node.isLink ? dirname(node.path) : node.path,
node.root.path,
true
)
}
if (node.integrity) {
n.integrity = node.integrity
}
return n
}
static get Entry () {
return YarnLockEntry
}
}
class YarnLockEntry {
#specs
constructor (specs) {
this.#specs = new Set(specs)
this.resolved = null
this.version = null
this.integrity = null
this.dependencies = null
this.optionalDependencies = null
}
toString () {
// sort objects to the bottom, then alphabetical
return ([...this.#specs]
.sort(localeCompare)
.map(quoteIfNeeded).join(', ') +
':\n' +
Object.getOwnPropertyNames(this)
.filter(prop => this[prop] !== null)
.sort(priorityThenLocaleCompare)
.map(prop =>
typeof this[prop] !== 'object'
? ` ${prop} ${prop === 'integrity' ? this[prop] : JSON.stringify(this[prop])}\n`
: Object.keys(this[prop]).length === 0 ? ''
: ` ${prop}:\n` + sortKV(this[prop]) + '\n')
.join('')).trim()
}
addSpec (spec) {
this.#specs.add(spec)
}
}
module.exports = YarnLock
@@ -0,0 +1,99 @@
{
"name": "@npmcli/arborist",
"version": "9.7.0",
"description": "Manage node_modules trees",
"dependencies": {
"@gar/promise-retry": "^1.0.0",
"@isaacs/string-locale-compare": "^1.1.0",
"@npmcli/fs": "^5.0.0",
"@npmcli/installed-package-contents": "^4.0.0",
"@npmcli/map-workspaces": "^5.0.0",
"@npmcli/metavuln-calculator": "^9.0.2",
"@npmcli/name-from-folder": "^4.0.0",
"@npmcli/node-gyp": "^5.0.0",
"@npmcli/package-json": "^7.0.0",
"@npmcli/query": "^5.0.0",
"@npmcli/redact": "^4.0.0",
"@npmcli/run-script": "^10.0.0",
"bin-links": "^6.0.0",
"cacache": "^20.0.1",
"common-ancestor-path": "^2.0.0",
"hosted-git-info": "^9.0.0",
"json-stringify-nice": "^1.1.4",
"lru-cache": "^11.2.1",
"minimatch": "^10.0.3",
"nopt": "^9.0.0",
"npm-install-checks": "^8.0.0",
"npm-package-arg": "^13.0.0",
"npm-pick-manifest": "^11.0.1",
"npm-registry-fetch": "^19.0.0",
"pacote": "^21.0.2",
"parse-conflict-json": "^5.0.1",
"proc-log": "^6.0.0",
"proggy": "^4.0.0",
"promise-all-reject-late": "^1.0.0",
"promise-call-limit": "^3.0.1",
"semver": "^7.3.7",
"ssri": "^13.0.0",
"treeverse": "^3.0.0",
"walk-up-path": "^4.0.0"
},
"devDependencies": {
"@npmcli/eslint-config": "^5.0.1",
"@npmcli/mock-registry": "^1.0.0",
"@npmcli/template-oss": "4.29.0",
"benchmark": "^2.1.4",
"minify-registry-metadata": "^4.0.0",
"nock": "^13.3.3",
"tap": "^16.3.8",
"tar-stream": "^3.0.0",
"tcompare": "^5.0.6"
},
"scripts": {
"test": "tap",
"posttest": "npm run lint",
"snap": "tap",
"test-proxy": "ARBORIST_TEST_PROXY=1 tap --snapshot",
"lint": "npm run eslint",
"lintfix": "npm run eslint -- --fix",
"benchmark": "node scripts/benchmark.js",
"benchclean": "rm -rf scripts/benchmark/*/",
"postlint": "template-oss-check",
"template-oss-apply": "template-oss-apply --force",
"eslint": "eslint \"**/*.{js,cjs,ts,mjs,jsx,tsx}\""
},
"repository": {
"type": "git",
"url": "git+https://github.com/npm/cli.git",
"directory": "workspaces/arborist"
},
"author": "GitHub Inc.",
"license": "ISC",
"files": [
"bin/",
"lib/"
],
"main": "lib/index.js",
"bin": {
"arborist": "bin/index.js"
},
"tap": {
"after": "test/fixtures/cleanup.js",
"test-env": [
"LC_ALL=sk"
],
"timeout": "720",
"nyc-arg": [
"--exclude",
"tap-snapshots/**"
]
},
"engines": {
"node": "^20.17.0 || >=22.9.0"
},
"templateOSS": {
"//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
"version": "4.29.0",
"content": "../../scripts/template-oss/index.js"
}
}
@@ -0,0 +1,15 @@
The ISC License
Copyright (c) npm, Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1,226 @@
# `@npmcli/config`
Configuration management for the npm cli.
This module is the spiritual descendant of [`npmconf`](http://npm.im/npmconf), and the code that once lived in npm's
`lib/config/` folder.
It does the management of configuration files that npm uses, but importantly, does _not_ define all the configuration defaults or types, as those parts make more sense to live within the npm CLI itself.
The only exceptions:
- The `prefix` config value has some special semantics, setting the local prefix if specified on the CLI options and not in global mode, or the global prefix otherwise.
- The `project` config file is loaded based on the local prefix (which can only be set by the CLI config options, and otherwise defaults to a walk up the folder tree to the first parent containing a `node_modules` folder, `package.json` file, or `package-lock.json` file.)
- The `userconfig` value, as set by the environment and CLI (defaulting to
`~/.npmrc`, is used to load user configs.
- The `globalconfig` value, as set by the environment, CLI, and
`userconfig` file (defaulting to `$PREFIX/etc/npmrc`) is used to load global configs.
- A `builtin` config, read from a `npmrc` file in the root of the npm project itself, overrides all defaults.
The resulting hierarchy of configs:
- CLI switches.
eg `--some-key=some-value` on the command line.
These are parsed by [`nopt`](http://npm.im/nopt), which is not a great choice, but it's the one that npm has used forever, and changing it will be difficult.
- Environment variables.
eg `npm_config_some_key=some_value` in the environment.
There is no way at this time to modify this prefix.
- INI-formatted project configs.
eg `some-key = some-value` in the
`localPrefix` folder (ie, the `cwd`, or its nearest parent that contains either a `node_modules` folder or `package.json` file.)
- INI-formatted userconfig file.
eg `some-key = some-value` in `~/.npmrc`.
The `userconfig` config value can be overridden by the `cli`, `env`, or
`project` configs to change this value.
- INI-formatted globalconfig file.
eg `some-key = some-value` in the `globalPrefix` folder, which is inferred by looking at the location of the node executable, or the `prefix` setting in the `cli`, `env`, `project`, or `userconfig`.
The `globalconfig` value at any of those levels can override this.
- INI-formatted builtin config file.
eg `some-key = some-value` in `/usr/local/lib/node_modules/npm/npmrc`.
This is not configurable, and is determined by looking in the `npmPath` folder.
- Default values (passed in by npm when it loads this module).
## USAGE
```js
const Config = require('@npmcli/config')
const { shorthands, definitions, flatten } = require('@npmcli/config/lib/definitions')
const conf = new Config({
// path to the npm module being run
npmPath: resolve(__dirname, '..'),
definitions,
shorthands,
flatten,
// optional, defaults to process.argv
// argv: [] <- if you are using this package in your own cli
// and don't want to have colliding argv
argv: process.argv,
// optional, defaults to process.env
env: process.env,
// optional, defaults to process.execPath
execPath: process.execPath,
// optional, defaults to process.platform
platform: process.platform,
// optional, defaults to process.cwd()
cwd: process.cwd(),
})
// emits log events on the process object
// see `proc-log` for more info
process.on('log', (level, ...args) => {
console.log(level, ...args)
})
// returns a promise that fails if config loading fails, and
// resolves when the config object is ready for action
conf.load().then(() => {
conf.validate()
console.log('loaded ok! some-key = ' + conf.get('some-key'))
}).catch(er => {
console.error('error loading configs!', er)
})
```
## API
The `Config` class is the sole export.
```js
const Config = require('@npmcli/config')
```
### static `Config.typeDefs`
The type definitions passed to `nopt` for CLI option parsing and known configuration validation.
### constructor `new Config(options)`
Options:
- `types` Types of all known config values.
Note that some are effectively given semantic value in the config loading process itself.
- `shorthands` An object mapping a shorthand value to an array of CLI arguments that replace it.
- `defaults` Default values for each of the known configuration keys.
These should be defined for all configs given a type, and must be valid.
- `npmPath` The path to the `npm` module, for loading the `builtin` config file.
- `cwd` Optional, defaults to `process.cwd()`, used for inferring the
`localPrefix` and loading the `project` config.
- `platform` Optional, defaults to `process.platform`.
Used when inferring the `globalPrefix` from the `execPath`, since this is done differently on Windows.
- `execPath` Optional, defaults to `process.execPath`.
Used to infer the
`globalPrefix`.
- `env` Optional, defaults to `process.env`.
Source of the environment variables for configuration.
- `argv` Optional, defaults to `process.argv`.
Source of the CLI options used for configuration.
Returns a `config` object, which is not yet loaded.
Fields:
- `config.globalPrefix` The prefix for `global` operations.
Set by the
`prefix` config value, or defaults based on the location of the
`execPath` option.
- `config.localPrefix` The prefix for `local` operations.
Set by the
`prefix` config value on the CLI only, or defaults to either the `cwd` or its nearest ancestor containing a `node_modules` folder or `package.json` file.
- `config.sources` A read-only `Map` of the file (or a comment, if no file found, or relevant) to the config level loaded from that source.
- `config.data` A `Map` of config level to `ConfigData` objects.
These objects should not be modified directly under any circumstances.
- `source` The source where this data was loaded from.
- `raw` The raw data used to generate this config data, as it was parsed initially from the environment, config file, or CLI options.
- `data` The data object reflecting the inheritance of configs up to this point in the chain.
- `loadError` Any errors encountered that prevented the loading of this config data.
- `config.list` A list sorted in priority of all the config data objects in the prototype chain.
`config.list[0]` is the `cli` level,
`config.list[1]` is the `env` level, and so on.
- `cwd` The `cwd` param
- `env` The `env` param
- `argv` The `argv` param
- `execPath` The `execPath` param
- `platform` The `platform` param
- `defaults` The `defaults` param
- `shorthands` The `shorthands` param
- `types` The `types` param
- `npmPath` The `npmPath` param
- `globalPrefix` The effective `globalPrefix`
- `localPrefix` The effective `localPrefix`
- `prefix` If `config.get('global')` is true, then `globalPrefix`, otherwise `localPrefix`
- `home` The user's home directory, found by looking at `env.HOME` or calling `os.homedir()`.
- `loaded` A boolean indicating whether or not configs are loaded
- `valid` A getter that returns `true` if all the config objects are valid.
Any data objects that have been modified with `config.set(...)` will be re-evaluated when `config.valid` is read.
### `config.load()`
Load configuration from the various sources of information.
Returns a `Promise` that resolves when configuration is loaded, and fails if a fatal error is encountered.
### `config.find(key)`
Find the effective place in the configuration levels a given key is set.
Returns one of: `cli`, `env`, `project`, `user`, `global`, `builtin`, or
`default`.
Returns `null` if the key is not set.
### `config.get(key, where = 'cli')`
Load the given key from the config stack.
### `config.set(key, value, where = 'cli')`
Set the key to the specified value, at the specified level in the config stack.
### `config.delete(key, where = 'cli')`
Delete the configuration key from the specified level in the config stack.
### `config.validate(where)`
Verify that all known configuration options are set to valid values, and log a warning if they are invalid.
Invalid auth options will cause this method to throw an error with a `code` property of `ERR_INVALID_AUTH`, and a `problems` property listing the specific concerns with the current configuration.
If `where` is not set, then all config objects are validated.
Returns `true` if all configs are valid.
Note that it's usually enough (and more efficient) to just check
`config.valid`, since each data object is marked for re-evaluation on every
`config.set()` operation.
### `config.repair(problems)`
Accept an optional array of problems (as thrown by `config.validate()`) and perform the necessary steps to resolve them.
If no problems are provided, this method will call `config.validate()` internally to retrieve them.
Note that you must `await config.save('user')` in order to persist the changes.
### `config.isDefault(key)`
Returns `true` if the value is coming directly from the default definitions, if the current value for the key config is coming from any other source, returns `false`.
This method can be used for avoiding or tweaking default values, e.g:
> Given a global default definition of foo='foo' it's possible to read that value such as:
>
> ```js
> const save = config.get('foo')
> ```
>
> Now in a different place of your app it's possible to avoid using the `foo` default value, by checking to see if the current config value is currently one that was defined by the default definitions:
>
> ```js
> const save = config.isDefault('foo') ? 'bar' : config.get('foo')
> ```
### `config.save(where)`
Save the config file specified by the `where` param.
Must be one of
`project`, `user`, `global`, `builtin`.
@@ -0,0 +1,255 @@
// class that describes a config key we know about
// this keeps us from defining a config key and not
// providing a default, description, etc.
//
// TODO: some kind of categorization system, so we can
// say "these are for registry access", "these are for
// version resolution" etc.
const required = ['type', 'description', 'default', 'key']
const allowed = [
'default',
'defaultDescription',
'deprecated',
'description',
'exclusive',
'flatten',
'hint',
'key',
'short',
'type',
'typeDescription',
'usage',
'envExport',
'alias',
'required',
]
const {
semver: { type: semver },
Umask: { type: Umask },
url: { type: url },
path: { type: path },
} = require('../type-defs.js')
class Definition {
constructor (key, def) {
this.key = key
// if it's set falsey, don't export it; otherwise, we do by default
this.envExport = true
Object.assign(this, def)
this.validate()
if (!this.defaultDescription) {
this.defaultDescription = describeValue(this.default)
}
if (!this.typeDescription) {
this.typeDescription = describeType(this.type)
}
// hint is only used for non-boolean values
if (!this.hint) {
if (this.type === Number) {
this.hint = '<number>'
} else {
this.hint = `<${this.key}>`
}
}
if (!this.usage) {
this.usage = describeUsage(this)
}
}
validate () {
for (const req of required) {
if (!Object.prototype.hasOwnProperty.call(this, req)) {
throw new Error(`config lacks ${req}: ${this.key}`)
}
}
if (!this.key) {
throw new Error(`config lacks key: ${this.key}`)
}
for (const field of Object.keys(this)) {
if (!allowed.includes(field)) {
throw new Error(`config defines unknown field ${field}: ${this.key}`)
}
}
}
// a textual description of this config, suitable for help output
describe () {
const description = unindent(this.description)
const noEnvExport = this.envExport
? ''
: `
This value is not exported to the environment for child processes.
`
const deprecated = !this.deprecated ? '' : `* DEPRECATED: ${unindent(this.deprecated)}\n`
/* eslint-disable-next-line max-len */
const exclusive = !this.exclusive ? '' : `\nThis config cannot be used with: \`${this.exclusive.join('`, `')}\``
return wrapAll(`#### \`${this.key}\`
* Default: ${unindent(this.defaultDescription)}
* Type: ${unindent(this.typeDescription)}
${deprecated}
${description}
${exclusive}
${noEnvExport}`)
}
}
const describeUsage = def => {
let key = ''
// Single type
if (!Array.isArray(def.type)) {
if (def.short) {
key = `-${def.short}|`
}
if (def.type === Boolean && def.default !== false) {
key = `${key}--no-${def.key}`
} else {
key = `${key}--${def.key}`
}
if (def.type !== Boolean) {
key = `${key} ${def.hint}`
}
return key
}
key = `--${def.key}`
if (def.short) {
key = `-${def.short}|--${def.key}`
}
// Multiple types
let types = def.type
const multiple = types.includes(Array)
const bool = types.includes(Boolean)
// null type means optional and doesn't currently affect usage output since
// all non-optional params have defaults so we render everything as optional
types = types.filter(t => t !== null && t !== Array && t !== Boolean)
if (!types.length) {
return key
}
let description
if (!types.some(t => typeof t !== 'string')) {
// Specific values, use specifics given
description = `<${types.filter(d => d).join('|')}>`
} else {
// Generic values, use hint
description = def.hint
}
if (bool) {
// Currently none of our multi-type configs with boolean values default to
// false so all their hints should show `--no-`, if we ever add ones that
// default to false we can branch the logic here
key = `--no-${def.key}|${key}`
}
const usage = `${key} ${description}`
if (multiple) {
return `${usage} [${usage} ...]`
} else {
return usage
}
}
const describeType = type => {
if (Array.isArray(type)) {
const descriptions = type.filter(t => t !== Array).map(t => describeType(t))
// [a] => "a"
// [a, b] => "a or b"
// [a, b, c] => "a, b, or c"
// [a, Array] => "a (can be set multiple times)"
// [a, Array, b] => "a or b (can be set multiple times)"
const last = descriptions.length > 1 ? [descriptions.pop()] : []
const oxford = descriptions.length > 1 ? ', or ' : ' or '
const words = [descriptions.join(', ')].concat(last).join(oxford)
const multiple = type.includes(Array) ? ' (can be set multiple times)' : ''
return `${words}${multiple}`
}
// Note: these are not quite the same as the description printed
// when validation fails. In that case, we want to give the user
// a bit more information to help them figure out what's wrong.
switch (type) {
case String:
return 'String'
case Number:
return 'Number'
case Umask:
return 'Octal numeric string in range 0000..0777 (0..511)'
case Boolean:
return 'Boolean'
case Date:
return 'Date'
case path:
return 'Path'
case semver:
return 'SemVer string'
case url:
return 'URL'
default:
return describeValue(type)
}
}
// if it's a string, quote it. otherwise, just cast to string.
const describeValue = val => (typeof val === 'string' ? JSON.stringify(val) : String(val))
const unindent = s => {
// get the first \n followed by a bunch of spaces, and pluck off
// that many spaces from the start of every line.
const match = s.match(/\n +/)
return !match ? s.trim() : s.split(match[0]).join('\n').trim()
}
const wrap = s => {
const cols = Math.min(Math.max(20, process.stdout.columns) || 80, 80) - 5
return unindent(s)
.split(/[ \n]+/)
.reduce((left, right) => {
const last = left.split('\n').pop()
const join = last.length && last.length + right.length > cols ? '\n' : ' '
return left + join + right
})
}
const wrapAll = s => {
let inCodeBlock = false
return s
.split('\n\n')
.map(block => {
if (inCodeBlock || block.startsWith('```')) {
inCodeBlock = !block.endsWith('```')
return block
}
if (block.charAt(0) === '*') {
return (
'* ' +
block
.slice(1)
.trim()
.split('\n* ')
.map(li => {
return wrap(li).replace(/\n/g, '\n ')
})
.join('\n* ')
)
} else {
return wrap(block)
}
})
.join('\n\n')
}
module.exports = Definition
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,88 @@
const definitions = require('./definitions.js')
// use the defined flattening function, and copy over any scoped
// registries and registry-specific "nerfdart" configs verbatim
//
// TODO: make these getters so that we only have to make dirty
// the thing that changed, and then flatten the fields that
// could have changed when a config.set is called.
//
// TODO: move nerfdart auth stuff into a nested object that
// is only passed along to paths that end up calling npm-registry-fetch.
const flatten = (obj, flat = {}) => {
for (const [key, val] of Object.entries(obj)) {
const def = definitions[key]
if (def && def.flatten) {
def.flatten(key, obj, flat)
} else if (/@.*:registry$/i.test(key) || /^\/\//.test(key)) {
flat[key] = val
}
}
return flat
}
const definitionProps = Object.entries(definitions)
.reduce((acc, [key, { short = [], default: d }]) => {
// can be either an array or string
for (const s of [].concat(short)) {
acc.shorthands[s] = [`--${key}`]
}
acc.defaults[key] = d
return acc
}, { shorthands: {}, defaults: {} })
// aliases where they get expanded into a completely different thing
// these are NOT supported in the environment or npmrc files, only
// expanded on the CLI.
// TODO: when we switch off of nopt, use an arg parser that supports
// more reasonable aliasing and short opts right in the definitions set.
const shorthands = {
'enjoy-by': ['--before'],
d: ['--loglevel', 'info'],
dd: ['--loglevel', 'verbose'],
ddd: ['--loglevel', 'silly'],
quiet: ['--loglevel', 'warn'],
q: ['--loglevel', 'warn'],
s: ['--loglevel', 'silent'],
silent: ['--loglevel', 'silent'],
verbose: ['--loglevel', 'verbose'],
desc: ['--description'],
help: ['--usage'],
local: ['--no-global'],
n: ['--no-yes'],
no: ['--no-yes'],
porcelain: ['--parseable'],
readonly: ['--read-only'],
reg: ['--registry'],
iwr: ['--include-workspace-root'],
ws: ['--workspaces'],
...definitionProps.shorthands,
}
// These are the configs that we can nerf-dart. Only _auth even has a config definition so we have to explicitly validate them here.
// This is used to validate during "npm config set" and to not warn on loading unknown configs when we see these.
const nerfDarts = [
'_auth', // Has a config
'_authToken', // Does not have a config
'_password', // Does not have a config
'certfile', // Does not have a config
'email', // Does not have a config
'keyfile', // Does not have a config
'username', // Does not have a config
]
const proxyEnv = [
'http_proxy',
'https_proxy',
'proxy',
'no_proxy',
]
module.exports = {
defaults: definitionProps.defaults,
definitions,
flatten,
nerfDarts,
proxyEnv,
shorthands,
}
@@ -0,0 +1,16 @@
// replace any ${ENV} values with the appropriate environ.
// optional "?" modifier can be used like this: ${ENV?} so in case of the variable being not defined, it evaluates into empty string.
const envExpr = /(?<!\\)(\\*)\$\{([^${}?]+)(\?)?\}/g
module.exports = (f, env) => f.replace(envExpr, (orig, esc, name, modifier) => {
const fallback = modifier === '?' ? '' : `$\{${name}}`
const val = env[name] !== undefined ? env[name] : fallback
// consume the escape chars that are relevant.
if (esc.length % 2) {
return orig.slice((esc.length + 1) / 2)
}
return (esc.slice(esc.length / 2)) + val
})
@@ -0,0 +1,23 @@
'use strict'
class ErrInvalidAuth extends Error {
constructor (problems) {
let message = 'Invalid auth configuration found: '
message += problems.map((problem) => {
// istanbul ignore else
if (problem.action === 'delete') {
return `\`${problem.key}\` is not allowed in ${problem.where} config`
} else if (problem.action === 'rename') {
return `\`${problem.from}\` must be renamed to \`${problem.to}\` in ${problem.where} config`
}
}).join(', ')
message += '\nPlease run `npm config fix` to repair your configuration.`'
super(message)
this.code = 'ERR_INVALID_AUTH'
this.problems = problems
}
}
module.exports = {
ErrInvalidAuth,
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,18 @@
const { URL } = require('node:url')
/**
* Maps a URL to an identifier.
*
* Name courtesy schiffertronix media LLC, a New Jersey corporation
*
* @param {String} uri The URL to be nerfed.
*
* @returns {String} A nerfed URL.
*/
module.exports = (url) => {
const parsed = new URL(url)
const from = `${parsed.protocol}//${parsed.host}${parsed.pathname}`
const rel = new URL('.', from)
const res = `//${rel.host}${rel.pathname}`
return res
}
@@ -0,0 +1,23 @@
// Parse an `allow-scripts` raw config value (string or array of strings)
// into a flat array of trimmed package-spec entries. Shared between the
// CLI/env layer (via the `allow-scripts` definition's `flatten`) and the
// package.json / .npmrc layer (in lib/utils/resolve-allow-scripts.js) so
// both paths agree on quoting, whitespace, and duplicate handling.
const parseAllowScriptsList = (raw) => {
const parts = []
const entries = Array.isArray(raw) ? raw : (typeof raw === 'string' ? [raw] : [])
for (const entry of entries) {
if (typeof entry !== 'string') {
continue
}
for (const part of entry.split(',')) {
const trimmed = part.trim()
if (trimmed) {
parts.push(trimmed)
}
}
}
return parts
}
module.exports = parseAllowScriptsList
@@ -0,0 +1,86 @@
// Parse a field, coercing it to the best type available.
const typeDefs = require('./type-defs.js')
const envReplace = require('./env-replace.js')
const { resolve } = require('node:path')
const { parse: umaskParse } = require('./umask.js')
const parseField = (f, key, opts, listElement = false) => {
if (typeof f !== 'string' && !Array.isArray(f)) {
return f
}
const { platform, types, home, env } = opts
// type can be array or a single thing. coerce to array.
const typeList = new Set([].concat(types[key]))
const isPath = typeList.has(typeDefs.path.type)
const isBool = typeList.has(typeDefs.Boolean.type)
const isString = isPath || typeList.has(typeDefs.String.type)
const isUmask = typeList.has(typeDefs.Umask.type)
const isNumber = typeList.has(typeDefs.Number.type)
const isList = !listElement && typeList.has(Array)
const isDate = typeList.has(typeDefs.Date.type)
if (Array.isArray(f)) {
return !isList ? f : f.map(field => parseField(field, key, opts, true))
}
// now we know it's a string
f = f.trim()
// list types get put in the environment separated by double-\n
// usually a single \n would suffice, but ca/cert configs can contain
// line breaks and multiple entries.
if (isList) {
return parseField(f.split('\n\n'), key, opts)
}
// --foo is like --foo=true for boolean types
if (isBool && !isString && f === '') {
return true
}
// string types can be the string 'true', 'false', etc.
// otherwise, parse these values out
if (!isString && !isPath && !isNumber) {
switch (f) {
case 'true': return true
case 'false': return false
case 'null': return null
case 'undefined': return undefined
}
}
f = envReplace(f, env)
if (isDate) {
return new Date(f)
}
if (isPath) {
const homePattern = platform === 'win32' ? /^~(\/|\\)/ : /^~\//
if (homePattern.test(f) && home) {
f = resolve(home, f.slice(2))
} else {
f = resolve(f)
}
}
if (isUmask) {
try {
return umaskParse(f)
} catch (er) {
// let it warn later when we validate
return f
}
}
if (isNumber && !isNaN(f)) {
f = +f
}
return f
}
module.exports = parseField
@@ -0,0 +1,113 @@
// Set environment variables for any non-default configs,
// so that they're already there when we run lifecycle scripts.
//
// See https://github.com/npm/rfcs/pull/90
// Return the env key if this is a thing that belongs in the env.
// Ie, if the key isn't a @scope, //nerf.dart, or _private,
// and the value is a string or array. Otherwise return false.
const envKey = (key, val) => {
return !/^[/@_]/.test(key) &&
(typeof envVal(val) === 'string') &&
`npm_config_${key.replace(/-/g, '_').toLowerCase()}`
}
const envVal = val => Array.isArray(val) ? val.map(v => envVal(v)).join('\n\n')
: val === null || val === undefined || val === false ? ''
: typeof val === 'object' ? null
: String(val)
const sameConfigValue = (def, val) =>
!Array.isArray(val) || !Array.isArray(def) ? def === val
: sameArrayValue(def, val)
const sameArrayValue = (def, val) => {
if (def.length !== val.length) {
return false
}
for (let i = 0; i < def.length; i++) {
/* istanbul ignore next - there are no array configs where the default
* is not an empty array, so this loop is a no-op, but it's the correct
* thing to do if we ever DO add a config like that. */
if (def[i] !== val[i]) {
return false
}
}
return true
}
const setEnv = (env, rawKey, rawVal) => {
const val = envVal(rawVal)
const key = envKey(rawKey, val)
if (key && val !== null) {
env[key] = val
}
}
const setEnvs = (config) => {
// This ensures that all npm config values that are not the defaults are
// shared appropriately with child processes, without false positives.
const {
env,
defaults,
definitions,
list: [cliConf, envConf],
} = config
env.INIT_CWD = process.cwd()
// if the key is deprecated, skip it always.
// if the key is the default value,
// if the environ is NOT the default value,
// set the environ
// else skip it, it's fine
// if the key is NOT the default value,
// if the env is setting it, then leave it (already set)
// otherwise, set the env
const cliSet = new Set(Object.keys(cliConf))
const envSet = new Set(Object.keys(envConf))
for (const key in cliConf) {
const { deprecated, envExport = true } = definitions[key] || {}
if (deprecated || envExport === false) {
continue
}
if (sameConfigValue(defaults[key], cliConf[key])) {
// config is the default, if the env thought different, then we
// have to set it BACK to the default in the environment.
if (!sameConfigValue(envConf[key], cliConf[key])) {
setEnv(env, key, cliConf[key])
}
} else {
// config is not the default. if the env wasn't the one to set
// it that way, then we have to put it in the env
if (!(envSet.has(key) && !cliSet.has(key))) {
setEnv(env, key, cliConf[key])
}
}
}
// also set some other common nice envs that we want to rely on
env.HOME = config.home
// TODO this may not be the best away to persist these
env.npm_config_global_prefix = config.globalPrefix
env.npm_config_local_prefix = config.localPrefix
if (cliConf.editor) {
env.EDITOR = cliConf.editor
}
// note: this doesn't affect the *current* node process, of course, since
// it's already started, but it does affect the options passed to scripts.
if (cliConf['node-options']) {
env.NODE_OPTIONS = cliConf['node-options']
}
// the node-gyp bin uses this so we always set it
env.npm_config_node_gyp = cliConf['node-gyp']
// this doesn't have a full definition so we manually export it here
env.npm_config_npm_version = cliConf['npm-version'] || 'unknown'
env.npm_execpath = config.npmBin
env.NODE = env.npm_node_execpath = config.execPath
}
module.exports = setEnvs
@@ -0,0 +1,61 @@
const nopt = require('nopt')
const { validate: validateUmask } = require('./umask.js')
class Umask {}
class Semver {}
const semverValid = require('semver/functions/valid')
const validateSemver = (data, k, val) => {
const valid = semverValid(val)
if (!valid) {
return false
}
data[k] = valid
}
const noptValidatePath = nopt.typeDefs.path.validate
const validatePath = (data, k, val) => {
if (typeof val !== 'string') {
return false
}
return noptValidatePath(data, k, val)
}
// add descriptions so we can validate more usefully
module.exports = {
...nopt.typeDefs,
semver: {
type: Semver,
validate: validateSemver,
description: 'full valid SemVer string',
},
Umask: {
type: Umask,
validate: validateUmask,
description: 'octal number in range 0o000..0o777 (0..511)',
},
url: {
...nopt.typeDefs.url,
description: 'full url with "http://"',
},
path: {
...nopt.typeDefs.path,
validate: validatePath,
description: 'valid filesystem path',
},
Number: {
...nopt.typeDefs.Number,
description: 'numeric value',
},
Boolean: {
...nopt.typeDefs.Boolean,
description: 'boolean value (true or false)',
},
Date: {
...nopt.typeDefs.Date,
description: 'valid Date string',
},
}
// TODO: make nopt less of a global beast so this kludge isn't necessary
nopt.typeDefs = module.exports
@@ -0,0 +1,21 @@
// return the description of the valid values of a field
// returns a string for one thing, or an array of descriptions
const typeDefs = require('./type-defs.js')
const typeDescription = t => {
if (!t || typeof t !== 'function' && typeof t !== 'object') {
return t
}
if (Array.isArray(t)) {
return t.map(t => typeDescription(t))
}
for (const { type, description } of Object.values(typeDefs)) {
if (type === t) {
return description || type
}
}
return t
}
module.exports = t => [].concat(typeDescription(t)).filter(t => t !== undefined)
@@ -0,0 +1,35 @@
const parse = val => {
// this is run via nopt and parse field where everything is
// converted to a string first, ignoring coverage for now
// instead of figuring out what is happening under the hood in nopt
// istanbul ignore else
if (typeof val === 'string') {
if (/^0o?[0-7]+$/.test(val)) {
return parseInt(val.replace(/^0o?/, ''), 8)
} else if (/^[1-9][0-9]*$/.test(val)) {
return parseInt(val, 10)
} else {
throw new Error(`invalid umask value: ${val}`)
}
} else {
if (typeof val !== 'number') {
throw new Error(`invalid umask value: ${val}`)
}
val = Math.floor(val)
if (val < 0 || val > 511) {
throw new Error(`invalid umask value: ${val}`)
}
return val
}
}
const validate = (data, k, val) => {
try {
data[k] = parse(val)
return true
} catch (er) {
return false
}
}
module.exports = { parse, validate }
Loaded 100 of 2001 files, more files were not shown because too many files have changed in this diff. Show more