mirror of
https://github.com/renovatebot/renovate.git
synced 2026-10-26 07:20:39 +00:00
a1971afa78
* ci(linter): add renovate/require-regex-util rule Adds a custom oxlint rule that flags raw RegExp usage in lib/ production code: any `new RegExp(...)` call and any regex literal that is not the direct first argument of `regEx(...)`. The regEx() helper from lib/util/regex.ts uses the RE2 engine for ReDoS-safe matching and caches compiled patterns. lib/util/regex.ts itself is exempt. The rule is disabled for **/*.spec.ts, test/**, and tools/** via config overrides, because vitest APIs such as expect.stringMatching need bare RegExp instances (a past commit reverted regEx() there). Severity is set to "warn" instead of "error": the repo currently has 197 pre-existing violations (above the agreed mechanical-fix threshold of 80), and a number of them are deliberately RE2-incompatible patterns (lookahead/lookbehind/named backreferences, see TODO #12872/#12875 markers in e.g. lib/modules/versioning/rez/ pattern.ts and lib/modules/manager/dockerfile/extract.ts) that need per-site judgment before they can be wrapped in regEx() or suppressed with a justified disable comment. No violation fixes are included in this commit. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * ci(linter): enforce renovate/require-regex-util at error Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor: use regEx() for regex literals and RegExp construction in lib Burn down all renovate/require-regex-util violations (197): - wrap RE2-compatible regex literals and RegExp constructions in regEx() - rewrite lookaround patterns where behavior is provably preserved: goproxy separator split, sbt-package link extraction, leiningen comment stripping and keyword search, npm GitHub owner validation (unit-count bound replaced by an explicit length check) - keep native RegExp with disable comments where RE2 cannot be used: moo / good-enough-parser lexer patterns (moo rejects RE2 instances), MigrationsService propertyName (matched via isRegExp, which RE2 instances fail), rez named backreferences and lookaheads, GitHub markdown link negative lookbehind, emoji astral-plane matching - drop stale TODO #12875 comments for patterns current re2 handles Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor: reduce require-regex-util disable comments Rewrite the github massage-markdown-links URL regex to capture an optional `api.` prefix in a named group (skipped in code) instead of relying on a negative lookbehind, so both the base and global variants can use regEx()/RE2 and their two disable comments are removed. Add spec cases for api-prefixed URLs that match the URL shape. Consolidate the four related line disables in rez/pattern.ts (bound and range patterns emulating Python conditional patterns via named backreferences and lookaheads, which RE2 cannot express) into a single block disable/enable pair, matching the treatment of the lexer files. Disable comment count drops from 13 to 8. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(datasource/sbt-package): use named capture group in link regex Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(manager/cake): restore original multiLineComment regex spelling The [^] -> [\\s\\S] rewrite was only needed for an RE2 wrap that was later reverted; keep the diff minimal. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(datasource/go): skip empty GOPROXY segments The switch from a lookahead split to a capture-group split (needed because RE2 has no lookahead support) changed how empty GOPROXY segments parse: `a|,b` yielded `{ url: ',', fallback: '|' }` before and `{ url: '', fallback: ',' }` after. The comma fallback only continues on 404/410, so a failing proxy ahead of an empty segment silently stopped the lookup instead of falling through to the remaining proxies. Empty segments carry no url to query, so drop them. * refactor(datasource/go): use isTruthy for empty segment filter * fix(datasource/go): filter empty segments before mapping Filtering after the map broke the contextual typing from the GoproxyItem[] annotation, widening fallback to string. * refactor(platform/github): use regEx() for merge queue query stripping --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
277 lines
6.9 KiB
TypeScript
277 lines
6.9 KiB
TypeScript
import {
|
|
isArray,
|
|
isBuffer,
|
|
isDate,
|
|
isEmptyObject,
|
|
isError,
|
|
isFunction,
|
|
isNonEmptyObject,
|
|
isNonEmptyStringAndNotWhitespace,
|
|
isObject,
|
|
isPlainObject,
|
|
isString,
|
|
} from '@sindresorhus/is';
|
|
import { RequestError as HttpError } from 'got';
|
|
import { DateTime } from 'luxon';
|
|
import { ZodError } from 'zod/v4';
|
|
import { ExecError } from '../util/exec/exec-error.ts';
|
|
import { regEx } from '../util/regex.ts';
|
|
import { redactedFields, sanitize } from '../util/sanitize.ts';
|
|
|
|
const contentFields = [
|
|
'content',
|
|
'contents',
|
|
'packageLockParsed',
|
|
'yarnLockParsed',
|
|
];
|
|
|
|
type ZodShortenedIssue =
|
|
| null
|
|
| string
|
|
| string[]
|
|
| {
|
|
[key: string]: ZodShortenedIssue;
|
|
};
|
|
|
|
export function prepareZodIssues(input: unknown): ZodShortenedIssue {
|
|
if (!isPlainObject(input)) {
|
|
return null;
|
|
}
|
|
|
|
let err: null | string | string[] = null;
|
|
// v8 ignore else -- TODO: add test #40625
|
|
if (isArray(input._errors, isString)) {
|
|
if (input._errors.length === 1) {
|
|
err = input._errors[0];
|
|
} else if (input._errors.length > 1) {
|
|
err = input._errors;
|
|
} else {
|
|
err = null;
|
|
}
|
|
}
|
|
delete input._errors;
|
|
|
|
if (isEmptyObject(input)) {
|
|
return err;
|
|
}
|
|
|
|
const output: Record<string, ZodShortenedIssue> = {};
|
|
const entries = Object.entries(input);
|
|
for (const [key, value] of entries.slice(0, 3)) {
|
|
const child = prepareZodIssues(value);
|
|
// v8 ignore else -- TODO: add test #40625
|
|
if (child !== null) {
|
|
output[key] = child;
|
|
}
|
|
}
|
|
|
|
if (entries.length > 3) {
|
|
output.___ = `... ${entries.length - 3} more`;
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
export function prepareZodError(err: ZodError): Record<string, unknown> {
|
|
Object.defineProperty(err, 'message', {
|
|
get: () => 'Schema error',
|
|
/* v8 ignore start -- TODO: drop set? */
|
|
set: () => {
|
|
/* intentionally empty */
|
|
},
|
|
// v8 ignore stop -- TODO: drop set? */
|
|
});
|
|
|
|
return {
|
|
message: err.message,
|
|
stack: err.stack,
|
|
issues: prepareZodIssues(err.format()),
|
|
};
|
|
}
|
|
|
|
export default function prepareError(err: Error): Record<string, unknown> {
|
|
if (err instanceof ZodError) {
|
|
return prepareZodError(err);
|
|
}
|
|
|
|
const response: Record<string, unknown> = {
|
|
// We want to loose class information, but keep enumerable properties of the error
|
|
// oxlint-disable-next-line typescript/no-misused-spread
|
|
...err,
|
|
};
|
|
|
|
// Required as message is non-enumerable
|
|
if (!response.message && err.message) {
|
|
response.message = err.message;
|
|
}
|
|
|
|
// Required as stack is non-enumerable
|
|
if (!response.stack && err.stack) {
|
|
response.stack = err.stack;
|
|
}
|
|
|
|
if (err instanceof AggregateError) {
|
|
response.errors = err.errors.map((error) => prepareError(error));
|
|
}
|
|
|
|
// handle rawExec error
|
|
if (err instanceof ExecError && isNonEmptyObject(err.options?.env)) {
|
|
const env = Object.keys(err.options.env);
|
|
response.options = { ...err.options, env };
|
|
}
|
|
|
|
// handle got error
|
|
if (err instanceof HttpError) {
|
|
const options: Record<string, unknown> = {
|
|
headers: structuredClone(err.options.headers),
|
|
url: err.options.url?.toString(),
|
|
hostType: err.options.context.hostType,
|
|
};
|
|
response.options = options;
|
|
|
|
options.username = err.options.username;
|
|
options.password = err.options.password;
|
|
options.method = err.options.method;
|
|
options.http2 = err.options.http2;
|
|
|
|
// v8 ignore else -- TODO: add test #40625
|
|
if (err.response) {
|
|
response.response = {
|
|
statusCode: err.response.statusCode,
|
|
statusMessage: err.response.statusMessage,
|
|
body:
|
|
err.name === 'TimeoutError'
|
|
? undefined
|
|
: structuredClone(err.response.body),
|
|
headers: structuredClone(err.response.headers),
|
|
httpVersion: err.response.httpVersion,
|
|
retryCount: err.response.retryCount,
|
|
};
|
|
}
|
|
}
|
|
|
|
return response;
|
|
}
|
|
|
|
type NestedValue = unknown[] | object;
|
|
|
|
function isNested(value: unknown): value is NestedValue {
|
|
return isArray(value) || isObject(value);
|
|
}
|
|
|
|
export function sanitizeValue(
|
|
value: unknown,
|
|
seen = new WeakMap<NestedValue, unknown>(),
|
|
): any {
|
|
if (isString(value)) {
|
|
return sanitize(sanitizeUrls(value));
|
|
}
|
|
|
|
// Handle boxed String objects (e.g. from `new String()`), which `isString`
|
|
// rejects but `isObject` would iterate character-by-character producing
|
|
// `{"0":"h","1":"e",...}` in log output.
|
|
if (value instanceof String) {
|
|
return sanitize(sanitizeUrls(value.toString()));
|
|
}
|
|
|
|
if (isDate(value)) {
|
|
return value;
|
|
}
|
|
|
|
if (DateTime.isDateTime(value)) {
|
|
return value.toISO();
|
|
}
|
|
|
|
if (isFunction(value)) {
|
|
return '[function]';
|
|
}
|
|
|
|
if (isBuffer(value)) {
|
|
return '[content]';
|
|
}
|
|
|
|
if (isError(value)) {
|
|
const err = prepareError(value);
|
|
return sanitizeValue(err, seen);
|
|
}
|
|
|
|
if (isArray(value)) {
|
|
const length = value.length;
|
|
const arrayResult = Array(length);
|
|
seen.set(value, arrayResult);
|
|
for (let idx = 0; idx < length; idx += 1) {
|
|
const val = value[idx];
|
|
arrayResult[idx] =
|
|
isNested(val) && seen.has(val)
|
|
? seen.get(val)
|
|
: sanitizeValue(val, seen);
|
|
}
|
|
return arrayResult;
|
|
}
|
|
|
|
if (isObject(value)) {
|
|
const objectResult: Record<string, any> = {};
|
|
seen.set(value, objectResult);
|
|
for (const [key, val] of Object.entries<any>(value)) {
|
|
let curValue: any;
|
|
if (!val) {
|
|
curValue = val;
|
|
} else if (redactedFields.includes(key)) {
|
|
// Do not mask/sanitize secrets templates
|
|
if (isString(val) && regEx(/^{{\s*secrets\..*}}$/).test(val)) {
|
|
curValue = val;
|
|
} else {
|
|
curValue = '***********';
|
|
}
|
|
} else if (contentFields.includes(key)) {
|
|
curValue = '[content]';
|
|
} else if (key === 'secrets') {
|
|
curValue = {};
|
|
Object.keys(val).forEach((secretKey) => {
|
|
curValue[secretKey] = '***********';
|
|
});
|
|
} else {
|
|
curValue = seen.has(val) ? seen.get(val) : sanitizeValue(val, seen);
|
|
}
|
|
|
|
const sanitizedKey = sanitizeValue(key, seen);
|
|
objectResult[sanitizedKey] = curValue;
|
|
}
|
|
|
|
return objectResult;
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
const urlRe = regEx(/[a-z]{3,9}:\/\/[^@/]+@[a-z0-9.-]+/gi);
|
|
const urlCredRe = regEx(/\/\/[^@]+@/g);
|
|
const dataUriCredRe = regEx(/^(data:[0-9a-z-]+\/[0-9a-z-]+;).+/i);
|
|
|
|
export function sanitizeUrls(text: string): string {
|
|
return text
|
|
.replace(urlRe, (url) => {
|
|
return url.replace(urlCredRe, '//**redacted**@');
|
|
})
|
|
.replace(dataUriCredRe, '$1**redacted**');
|
|
}
|
|
|
|
export function getEnv(key: string): string | undefined {
|
|
return [process.env[`RENOVATE_${key}`], process.env[key]]
|
|
.map((v) => v?.toLowerCase().trim())
|
|
.find(isNonEmptyStringAndNotWhitespace);
|
|
}
|
|
|
|
export function getMessage(
|
|
p1: string | Record<string, any>,
|
|
p2?: string,
|
|
): string | undefined {
|
|
return isString(p1) ? p1 : p2;
|
|
}
|
|
|
|
export function toMeta(
|
|
p1: string | Record<string, any>,
|
|
): Record<string, unknown> {
|
|
return isObject(p1) ? p1 : {};
|
|
}
|