mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
31e1e42ad7
Lands gateway/integration on main: the guard rails and the domain decoupling for replacing s3s with rustfs/gateway (rustfs/backlog#2734, Phase 0 and Phase 1). The S3 edge is still s3s and no wire behaviour change is intended. - Guard rails: lower-only s3s footprint baseline and edge allow-list, per-crate test-count floor, 39 byte-level goldens for the persisted bucket-config XML families, a two-server differential harness over all 73 S3 operations, a JUnit differ, and a reference-parity lane against the sha256-pinned 1.0.5 binary. - Owned types: rustfs-s3-types (S3Error, S3ErrorCode, Body), RequestEnvelope, AuthzFacts. - Engine crates off s3s: ecstore, lifecycle, replication, scanner, protocols, s3select, s3-client, rpc. - Admin: error types moved; 300 of 314 Operation impls take the owned AdminRequest. - Server: the s3s-compat tower layers grouped under server/legacy_compat; RUSTFS_S3_STACK=legacy|gateway selector, where gateway refuses to start until the gateway edge exists. Fixes made on the branch that change ecstore and heal behaviour are listed in the pull request description. The per-commit history stays on gateway/integration.
1627 lines
63 KiB
Python
1627 lines
63 KiB
Python
#!/usr/bin/env python3
|
|
"""Codemod for the T1.12 admin boundary migration (rustfs/backlog#2788).
|
|
|
|
Rewrites admin handler files under ``rustfs/src/admin/handlers/**`` from the
|
|
``s3s`` request/response types (``S3Request<Body>`` / ``S3Response<(StatusCode,
|
|
Body)>`` / ``crate::admin::router::Operation``) to the owned boundary defined in
|
|
``rustfs/src/admin/operation.rs`` (``AdminRequest`` / ``AdminResponse`` /
|
|
``crate::admin::operation::Operation``).
|
|
|
|
Usage::
|
|
|
|
python3 scripts/dev_t112_admin_codemod.py [--check] [--credential-methods] [--rustfmt] <file.rs>...
|
|
|
|
``--check`` prints what would change (per-file line deltas and the TODO list)
|
|
without writing anything. ``--credential-methods`` additionally rewrites the
|
|
obvious ``cred.access_key`` / ``cred.secret_key`` field accesses on bindings
|
|
taken from ``req.credentials()`` into the ``RequestCredentials`` accessor
|
|
methods; by default those sites only get a TODO. ``--rustfmt`` runs
|
|
``rustfmt --edition 2024`` on every file that was written (syntax only, no
|
|
compilation).
|
|
|
|
The script is idempotent: running it again on an already migrated file makes no
|
|
further change, and TODO comments are never duplicated.
|
|
|
|
Everything the script cannot rewrite mechanically is left in place with a
|
|
``// TODO(T1.12-codemod): <reason>`` line inserted directly above it. Every
|
|
TODO (new or pre-existing) is listed on stdout as ``file:line: reason``.
|
|
|
|
Rewrite rules (see the per-phase functions below for the exact shapes):
|
|
|
|
1. imports: ``Operation`` leaves ``crate::admin::router`` imports and the owned
|
|
names are imported from ``crate::admin::operation``; ``Body``/``S3Request``/
|
|
``S3Response`` leave ``s3s``/``storage_api::s3`` imports; ``Body`` is imported
|
|
from ``rustfs_s3_types`` when still used; ``s3s::header::*`` becomes
|
|
``http::header::*``; ``LegacyAdminAdapter`` statics are removed and routes
|
|
register the handler directly.
|
|
2. signatures: ``S3Request<Body>`` -> ``AdminRequest``,
|
|
``S3Response<(StatusCode, Body)>`` -> ``AdminResponse``; ``mut`` on by-value
|
|
``AdminRequest`` parameters is kept only when the body needs it.
|
|
3. request fields: ``req.uri``/``req.headers``/``req.method``/``req.credentials``
|
|
become accessor calls; ``req.extensions`` lookups become ``remote_addr()``,
|
|
``req_info()``, ``req_info_mut()``, ``install_req_info()``, ``client_info()``,
|
|
``request_context()``; ``req.region``/``req.service`` and the remaining
|
|
``req.extensions`` uses get a TODO.
|
|
4. helpers: ``authorize_admin_request(&req, ..)`` ->
|
|
``authorize_admin_envelope(req.envelope(), ..)``; ``*_from_req(&req)`` ->
|
|
``*_from_envelope(req.envelope())``; ``*_from_extensions(&req.extensions`` ->
|
|
``*_from_envelope(req.envelope()``; ``KmsAdminAudit::from_request(..)`` ->
|
|
``from_envelope(req.envelope(), ..)``; ``CallerIdentity::resolve(&req)`` ->
|
|
``resolve_envelope(req.envelope())``; ``AccountAudit*::from_request(&req)`` ->
|
|
``from_envelope(req.envelope())``. Import lines follow the renames.
|
|
5. request body: ``req.input`` / ``std::mem::take(&mut req.input)`` ->
|
|
``req.take_body()``; assignments to ``req.input`` get a TODO.
|
|
6. responses: ``S3Response::new((A, B))`` -> ``AdminResponse::new(A, B)``;
|
|
``S3Response::with_headers((A, B), H)`` -> ``AdminResponse::with_headers(A, B, H)``;
|
|
other shapes and response field access get a TODO.
|
|
7. tests: ``S3Request { .. }`` literals become
|
|
``crate::admin::operation::test_support::TestRequest::new(method, uri)
|
|
.headers(..).extensions(..).body(..)[.credentials(a, b, None)].build()``.
|
|
Literals that feed the s3s-facing ``S3Router`` directly (``router.call`` /
|
|
``check_access``) and literals in production code are left alone, together
|
|
with the function that owns them, under a TODO.
|
|
8. string literals in source-inspecting tests follow the helper renames; any
|
|
remaining mention of a pre-migration symbol gets a TODO.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import difflib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
TODO_PREFIX = "// TODO(T1.12-codemod): "
|
|
SENT_OPEN = "\x01"
|
|
SENT_CLOSE = "\x02"
|
|
MAX_WIDTH = 130
|
|
|
|
# --------------------------------------------------------------------------- reasons
|
|
|
|
R_REGION = "request region/service is now Option<&str> via region()/service(); adjust the caller"
|
|
R_EXT = "request extensions are private on AdminRequest; use an accessor (remote_addr/req_info/client_info/request_context) or envelope()"
|
|
R_URI_ASSIGN = "assigning the request URI is not supported on AdminRequest"
|
|
R_HDR_MUT = "mutating request headers needs envelope_mut() on AdminRequest"
|
|
R_METHOD_ASSIGN = "assigning the request method is not supported on AdminRequest"
|
|
R_CRED_ASSIGN = "assigning request credentials is not supported on AdminRequest; build the request with credentials instead"
|
|
R_CRED_FIELD = "RequestCredentials exposes access_key()/secret_key()/session_token() methods (secret_key() returns &str) instead of fields"
|
|
R_BORROW = "AdminRequest accessors borrow (uri()/headers()/credentials()); clone or change the callee if it needs ownership"
|
|
R_INPUT_ASSIGN = "assigning the request body is not supported on AdminRequest; build the request with the body instead"
|
|
R_RESP_CTOR = "S3Response::new/with_headers with a non-(StatusCode, Body) output has no AdminResponse equivalent"
|
|
R_RESP_TYPE = "S3Response with a non-(StatusCode, Body) output has no AdminResponse equivalent"
|
|
R_REQ_TYPE = "S3Request with a non-Body input has no AdminRequest equivalent"
|
|
R_RESP_FIELD = "AdminResponse fields are private; use with_headers()/into_parts() instead of field access"
|
|
R_OUTPUT = "S3Response output tuple is gone; destructure AdminResponse::into_parts() instead"
|
|
R_LITERAL = "S3Request literal has an unsupported field value; build it with test_support::TestRequest manually"
|
|
R_LITERAL_REGION = "TestRequest region/service value is not None; verify the builder accepts it"
|
|
R_ROUTER = "router-level test drives the s3s-facing S3Router; keep s3s request types until the router boundary moves"
|
|
R_PROD_LITERAL = "production code builds an s3s S3Request by hand; construct AdminRequest::new(RequestEnvelope, Body) or keep the s3s type until the generic auth path moves"
|
|
R_STRING = "string literal references a pre-migration symbol; update the source-inspecting test"
|
|
|
|
# --------------------------------------------------------------------------- scanner
|
|
|
|
_TOKEN_RE = re.compile(r"//[^\n]*|/\*|(?<![\w])b?r#*\"|(?<![\w])b\"|\"|'(?:\\[^']*|[^'\\])'|[()\[\]{}]")
|
|
|
|
|
|
class Scanner:
|
|
"""Marks comment/string regions and pairs brackets for a Rust source text."""
|
|
|
|
def __init__(self, text: str) -> None:
|
|
self.text = text
|
|
n = len(text)
|
|
mask = bytearray(b"\x01") * n # 1 = code
|
|
strings: list[tuple[int, int]] = []
|
|
pairs: dict[int, int] = {}
|
|
stack: list[int] = []
|
|
pos = 0
|
|
while True:
|
|
m = _TOKEN_RE.search(text, pos)
|
|
if m is None:
|
|
break
|
|
tok = m.group(0)
|
|
i = m.start()
|
|
if tok.startswith("//"):
|
|
j = m.end()
|
|
mask[i:j] = bytes(j - i)
|
|
pos = j
|
|
elif tok == "/*":
|
|
j = _skip_block_comment(text, i)
|
|
mask[i:j] = bytes(j - i)
|
|
pos = j
|
|
elif tok.endswith('"'):
|
|
if "r" in tok:
|
|
hashes = tok.count("#")
|
|
close = '"' + "#" * hashes
|
|
k = text.find(close, m.end())
|
|
j = n if k < 0 else k + len(close)
|
|
else:
|
|
j = _skip_plain_string(text, m.end() - 1)
|
|
mask[i:j] = bytes(j - i)
|
|
strings.append((i, j))
|
|
pos = j
|
|
elif tok.startswith("'"):
|
|
j = m.end()
|
|
mask[i:j] = bytes(j - i)
|
|
pos = j
|
|
else:
|
|
if tok in "([{":
|
|
stack.append(i)
|
|
elif stack:
|
|
o = stack.pop()
|
|
pairs[o] = i
|
|
pairs[i] = o
|
|
pos = m.end()
|
|
self.mask = mask
|
|
self.pairs = pairs
|
|
self.strings = strings
|
|
|
|
def is_code(self, i: int) -> bool:
|
|
return 0 <= i < len(self.mask) and self.mask[i] == 1
|
|
|
|
def match(self, i: int) -> int | None:
|
|
return self.pairs.get(i)
|
|
|
|
def next_code(self, pattern: re.Pattern[str], start: int) -> re.Match[str] | None:
|
|
pos = start
|
|
while True:
|
|
m = pattern.search(self.text, pos)
|
|
if m is None:
|
|
return None
|
|
if self.is_code(m.start()):
|
|
return m
|
|
pos = m.end() if m.end() > m.start() else m.start() + 1
|
|
|
|
|
|
def _skip_block_comment(text: str, i: int) -> int:
|
|
depth = 1
|
|
j = i + 2
|
|
n = len(text)
|
|
while j < n and depth:
|
|
if text.startswith("/*", j):
|
|
depth += 1
|
|
j += 2
|
|
elif text.startswith("*/", j):
|
|
depth -= 1
|
|
j += 2
|
|
else:
|
|
j += 1
|
|
return j
|
|
|
|
|
|
def _skip_plain_string(text: str, i: int) -> int:
|
|
"""``i`` points at the opening quote; returns the index after the closing quote."""
|
|
j = i + 1
|
|
n = len(text)
|
|
while j < n:
|
|
c = text[j]
|
|
if c == "\\":
|
|
j += 2
|
|
continue
|
|
if c == '"':
|
|
return j + 1
|
|
j += 1
|
|
return n
|
|
|
|
|
|
# --------------------------------------------------------------------------- text utils
|
|
|
|
|
|
def line_start(text: str, pos: int) -> int:
|
|
return text.rfind("\n", 0, pos) + 1
|
|
|
|
|
|
def line_end(text: str, pos: int) -> int:
|
|
j = text.find("\n", pos)
|
|
return len(text) if j < 0 else j
|
|
|
|
|
|
def indent_of(text: str, pos: int) -> str:
|
|
ls = line_start(text, pos)
|
|
m = re.match(r"[ \t]*", text[ls:])
|
|
return m.group(0) if m else ""
|
|
|
|
|
|
def add_sentinel(text: str, pos: int, reason: str) -> str:
|
|
"""Attach a TODO sentinel to the end of the line containing ``pos``."""
|
|
le = line_end(text, pos)
|
|
token = f"{SENT_OPEN}{reason}{SENT_CLOSE}"
|
|
if token in text[line_start(text, pos):le]:
|
|
return text
|
|
return text[:le] + token + text[le:]
|
|
|
|
|
|
def materialize_todos(text: str) -> str:
|
|
"""Turn end-of-line sentinels into TODO comment lines above their line."""
|
|
if SENT_OPEN not in text:
|
|
return text
|
|
out_lines: list[str] = []
|
|
for line in text.split("\n"):
|
|
reasons: list[str] = []
|
|
for m in re.finditer(f"{SENT_OPEN}(.*?){SENT_CLOSE}", line):
|
|
if m.group(1) not in reasons:
|
|
reasons.append(m.group(1))
|
|
if not reasons:
|
|
out_lines.append(line)
|
|
continue
|
|
clean = re.sub(f"{SENT_OPEN}.*?{SENT_CLOSE}", "", line)
|
|
indent = re.match(r"[ \t]*", clean).group(0)
|
|
existing: list[str] = []
|
|
k = len(out_lines) - 1
|
|
while k >= 0 and out_lines[k].strip().startswith(TODO_PREFIX.strip()):
|
|
existing.append(out_lines[k].strip())
|
|
k -= 1
|
|
for reason in reasons:
|
|
todo = f"{TODO_PREFIX}{reason}"
|
|
if todo not in existing:
|
|
out_lines.append(f"{indent}{todo}")
|
|
out_lines.append(clean)
|
|
return "\n".join(out_lines)
|
|
|
|
|
|
def split_top_level(sc: Scanner, start: int, end: int) -> list[tuple[int, int]]:
|
|
"""Split ``text[start:end]`` at top-level commas; returns (s, e) spans."""
|
|
spans: list[tuple[int, int]] = []
|
|
depth = 0
|
|
i = start
|
|
item_start = start
|
|
text = sc.text
|
|
while i < end:
|
|
if not sc.is_code(i):
|
|
i += 1
|
|
continue
|
|
c = text[i]
|
|
if c in "([{":
|
|
depth += 1
|
|
elif c in ")]}":
|
|
depth -= 1
|
|
elif c == "," and depth == 0:
|
|
spans.append((item_start, i))
|
|
item_start = i + 1
|
|
i += 1
|
|
if text[item_start:end].strip():
|
|
spans.append((item_start, end))
|
|
return spans
|
|
|
|
|
|
def strip_span(text: str, span: tuple[int, int]) -> str:
|
|
return text[span[0]:span[1]].strip()
|
|
|
|
|
|
# --------------------------------------------------------------------------- function spans
|
|
|
|
|
|
@dataclass
|
|
class FnSpan:
|
|
name: str
|
|
ordinal: int # index among same-named functions, in file order
|
|
sig_start: int
|
|
params: tuple[int, int] # inside the parens
|
|
body: tuple[int, int] # inside the braces
|
|
|
|
@property
|
|
def key(self) -> tuple[str, int]:
|
|
return (self.name, self.ordinal)
|
|
|
|
|
|
_FN_RE = re.compile(r"\bfn\s+(\w+)\s*(<|\()")
|
|
|
|
|
|
def iter_fns(sc: Scanner) -> list[FnSpan]:
|
|
text = sc.text
|
|
out: list[FnSpan] = []
|
|
counts: Counter[str] = Counter()
|
|
pos = 0
|
|
while True:
|
|
m = sc.next_code(_FN_RE, pos)
|
|
if m is None:
|
|
break
|
|
pos = m.end()
|
|
i = m.end() - 1
|
|
if text[i] == "<":
|
|
depth = 0
|
|
while i < len(text):
|
|
if sc.is_code(i):
|
|
if text[i] == "<":
|
|
depth += 1
|
|
elif text[i] == ">":
|
|
depth -= 1
|
|
if depth == 0:
|
|
i += 1
|
|
break
|
|
i += 1
|
|
while i < len(text) and text[i] != "(":
|
|
i += 1
|
|
if i >= len(text):
|
|
break
|
|
p_open = i
|
|
p_close = sc.match(p_open)
|
|
if p_close is None:
|
|
continue
|
|
j = p_close + 1
|
|
body_open = None
|
|
while j < len(text):
|
|
if sc.is_code(j):
|
|
if text[j] == ";":
|
|
break
|
|
if text[j] == "{":
|
|
body_open = j
|
|
break
|
|
j += 1
|
|
if body_open is None:
|
|
continue
|
|
body_close = sc.match(body_open)
|
|
if body_close is None:
|
|
continue
|
|
name = m.group(1)
|
|
out.append(FnSpan(name, counts[name], m.start(), (p_open + 1, p_close), (body_open + 1, body_close)))
|
|
counts[name] += 1
|
|
return out
|
|
|
|
|
|
def enclosing_fn(fns: list[FnSpan], pos: int) -> FnSpan | None:
|
|
inside = [f for f in fns if f.body[0] <= pos < f.body[1]]
|
|
if not inside:
|
|
return None
|
|
return min(inside, key=lambda f: f.body[1] - f.body[0])
|
|
|
|
|
|
# --------------------------------------------------------------------------- context
|
|
|
|
|
|
@dataclass
|
|
class Ctx:
|
|
path: Path
|
|
credential_methods: bool
|
|
frozen: dict[tuple[str, int], str] = field(default_factory=dict) # fn key -> reason
|
|
adapter_map: dict[str, str] = field(default_factory=dict)
|
|
|
|
def frozen_spans(self, sc: Scanner, fns: list[FnSpan] | None = None) -> list[tuple[int, int]]:
|
|
if not self.frozen:
|
|
return []
|
|
fns = fns if fns is not None else iter_fns(sc)
|
|
return [(f.sig_start, f.body[1]) for f in fns if f.key in self.frozen]
|
|
|
|
|
|
def in_spans(pos: int, spans: list[tuple[int, int]]) -> bool:
|
|
return any(s <= pos < e for s, e in spans)
|
|
|
|
|
|
_ROUTER_CALL_RE = re.compile(r"\.check_access\(|\brouter\s*\.\s*call\(")
|
|
_LITERAL_RE = re.compile(r"(?<![\w:])(?:s3s::)?S3Request\s*\{")
|
|
_TEST_ATTR_RE = re.compile(r"#\[(?:tokio::)?test\]|#\[cfg\(test\)\]|#\[serial\]")
|
|
|
|
|
|
def test_region_start(text: str, path: Path) -> int:
|
|
"""Offset where test-only code starts (whole file for ``tests.rs`` modules)."""
|
|
if path.name == "tests.rs" or "/tests/" in path.as_posix():
|
|
return 0
|
|
m = re.search(r"^#\[cfg\(test\)\]\s*\n(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+", text, re.M)
|
|
return m.start() if m else len(text)
|
|
|
|
|
|
def is_test_code(text: str, pos: int, fns: list[FnSpan], test_start: int) -> bool:
|
|
if pos >= test_start:
|
|
return True
|
|
f = enclosing_fn(fns, pos)
|
|
if f is None:
|
|
return False
|
|
above = text[max(0, line_start(text, f.sig_start) - 200):f.sig_start]
|
|
return _TEST_ATTR_RE.search(above) is not None
|
|
|
|
|
|
def detect_frozen_fns(text: str, path: Path) -> dict[tuple[str, int], str]:
|
|
"""Functions that must keep the s3s request types, with the reason."""
|
|
sc = Scanner(text)
|
|
fns = iter_fns(sc)
|
|
frozen: dict[tuple[str, int], str] = {}
|
|
for f in fns:
|
|
if _ROUTER_CALL_RE.search(text[f.body[0]:f.body[1]]):
|
|
frozen[f.key] = R_ROUTER
|
|
builders = {m.group(1) for m in re.finditer(r"\brouter\s*\.\s*call\(\s*(\w+)\(", text)}
|
|
for f in fns:
|
|
if f.name in builders:
|
|
frozen[f.key] = R_ROUTER
|
|
test_start = test_region_start(text, path)
|
|
pos = 0
|
|
while True:
|
|
m = sc.next_code(_LITERAL_RE, pos)
|
|
if m is None:
|
|
break
|
|
pos = m.end()
|
|
if is_test_code(text, m.start(), fns, test_start):
|
|
continue
|
|
f = enclosing_fn(fns, m.start())
|
|
if f is not None and f.key not in frozen:
|
|
frozen[f.key] = R_PROD_LITERAL
|
|
return frozen
|
|
|
|
|
|
# --------------------------------------------------------------------------- request identifiers
|
|
|
|
|
|
_REQ_PARAM_RE = re.compile(r"(?<![\w&])(?:mut\s+)?(\w+)\s*:\s*&?\s*(?:mut\s+)?(?:s3s::)?(?:S3Request<[^>]*>|AdminRequest)\b")
|
|
|
|
|
|
def request_scopes(sc: Scanner, fns: list[FnSpan]) -> list[tuple[str, int, int]]:
|
|
"""Identifiers typed as the admin request, with the span they are valid in."""
|
|
scopes: list[tuple[str, int, int]] = []
|
|
for f in fns:
|
|
params = sc.text[f.params[0]:f.params[1]]
|
|
for m in _REQ_PARAM_RE.finditer(params):
|
|
if m.group(1) != "self":
|
|
scopes.append((m.group(1), f.sig_start, f.body[1]))
|
|
return scopes
|
|
|
|
|
|
def is_request_ident(name: str, pos: int, scopes: list[tuple[str, int, int]]) -> bool:
|
|
if name == "req":
|
|
return True
|
|
return any(n == name and s <= pos < e for n, s, e in scopes)
|
|
|
|
|
|
# --------------------------------------------------------------------------- rewrite helpers
|
|
|
|
|
|
def rewrite(text: str, pattern: re.Pattern[str], fn, *, only_code: bool = True, ctx: Ctx | None = None) -> str:
|
|
"""Apply ``fn(match, sc, scopes) -> (replacement|None, reason|None)`` back to front."""
|
|
sc = Scanner(text)
|
|
fns = iter_fns(sc)
|
|
frozen = ctx.frozen_spans(sc, fns) if ctx else []
|
|
scopes = request_scopes(sc, fns)
|
|
out = text
|
|
for m in reversed(list(pattern.finditer(text))):
|
|
if only_code and not sc.is_code(m.start()):
|
|
continue
|
|
if frozen and in_spans(m.start(), frozen):
|
|
continue
|
|
res = fn(m, sc, scopes)
|
|
if res is None:
|
|
continue
|
|
repl, reason = res
|
|
if repl is not None:
|
|
out = out[:m.start()] + repl + out[m.end():]
|
|
if reason:
|
|
out = add_sentinel(out, m.start(), reason)
|
|
return materialize_todos(out)
|
|
|
|
|
|
def simple_sub(text: str, pattern: str, repl: str, *, ctx: Ctx | None = None, only_code: bool = True) -> str:
|
|
return rewrite(text, re.compile(pattern), lambda m, sc, scopes: (m.expand(repl), None), ctx=ctx, only_code=only_code)
|
|
|
|
|
|
def req_sub(text: str, pattern: str, repl: str | None, reason: str | None, ctx: Ctx, *, group: int = 1) -> str:
|
|
"""Substitute ``pattern`` whose ``group`` captures the request identifier."""
|
|
|
|
def fn(m, sc, scopes):
|
|
if not is_request_ident(m.group(group), m.start(group), scopes):
|
|
return None
|
|
return (m.expand(repl) if repl is not None else None, reason)
|
|
|
|
return rewrite(text, re.compile(pattern), fn, ctx=ctx)
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: request literals (tests)
|
|
|
|
|
|
_STR = r"\"(?:[^\"\\]|\\.)*\""
|
|
|
|
|
|
def _unwrap_uri(expr: str) -> str:
|
|
m = re.fullmatch(rf"(?:http::)?Uri::from_static\(\s*({_STR})\s*\)", expr, re.S)
|
|
if m:
|
|
return m.group(1)
|
|
m = re.fullmatch(rf"({_STR})\s*\.parse\(\)\s*\.(?:unwrap|expect)\([^()]*\)", expr, re.S)
|
|
if m:
|
|
return m.group(1)
|
|
# A non-literal `.parse()` lost the `uri: Uri` field annotation that drove
|
|
# type inference; make the target type explicit.
|
|
m = re.fullmatch(r"(.*?)\.parse\(\)(\s*\.(?:unwrap|expect)\([^()]*\))?", expr, re.S)
|
|
if m and "::<" not in expr:
|
|
return f"{m.group(1)}.parse::<http::Uri>(){m.group(2) or ''}"
|
|
return expr
|
|
|
|
|
|
def _unwrap_into(expr: str) -> str:
|
|
"""Strip the owned-string conversions a literal does not need: the owned
|
|
``TestRequest::credentials`` takes ``&str``."""
|
|
while True:
|
|
m = re.fullmatch(rf"({_STR})\s*\.(?:into|to_string|to_owned)\(\)", expr, re.S)
|
|
if m is None:
|
|
m = re.fullmatch(rf"String::from\(\s*({_STR})\s*\)", expr, re.S)
|
|
if m is None:
|
|
return expr
|
|
expr = m.group(1)
|
|
|
|
|
|
def _unwrap_secret(expr: str) -> str:
|
|
m = re.fullmatch(r"(?:s3s::auth::|s3_auth::)?SecretKey::from\((.*)\)", expr, re.S)
|
|
return _unwrap_into(m.group(1).strip()) if m else _unwrap_into(expr)
|
|
|
|
|
|
_CRED_LIT_RE = re.compile(r"\s*Some\(\s*(?:s3s::auth::|s3_auth::)?Credentials\s*\{")
|
|
|
|
|
|
def _parse_credentials(sc: Scanner, span: tuple[int, int]) -> tuple[str, str] | None:
|
|
text = sc.text
|
|
m = _CRED_LIT_RE.match(text, span[0])
|
|
if not m or m.end() > span[1]:
|
|
return None
|
|
open_brace = m.end() - 1
|
|
close = sc.match(open_brace)
|
|
if close is None:
|
|
return None
|
|
fields: dict[str, str] = {}
|
|
for s, e in split_top_level(sc, open_brace + 1, close):
|
|
item = text[s:e].strip()
|
|
fm = re.match(r"(\w+)\s*:\s*(.*)$", item, re.S)
|
|
if fm:
|
|
fields[fm.group(1)] = fm.group(2).strip()
|
|
elif re.fullmatch(r"\w+", item):
|
|
fields[item] = item
|
|
if set(fields) != {"access_key", "secret_key"}:
|
|
return None
|
|
return (_unwrap_into(fields["access_key"]), _unwrap_secret(fields["secret_key"]))
|
|
|
|
|
|
def _reindent(expr: str, old_indent: str, new_indent: str) -> str:
|
|
lines = expr.split("\n")
|
|
if len(lines) == 1:
|
|
return expr
|
|
out = [lines[0]]
|
|
for ln in lines[1:]:
|
|
if ln.startswith(old_indent):
|
|
ln = new_indent + ln[len(old_indent):]
|
|
out.append(ln)
|
|
return "\n".join(out)
|
|
|
|
|
|
def convert_request_literals(text: str, ctx: Ctx) -> str:
|
|
pos = 0
|
|
while True:
|
|
sc = Scanner(text)
|
|
m = sc.next_code(_LITERAL_RE, pos)
|
|
if m is None:
|
|
break
|
|
open_brace = m.end() - 1
|
|
close_brace = sc.match(open_brace)
|
|
if close_brace is None:
|
|
pos = m.end()
|
|
continue
|
|
if in_spans(m.start(), ctx.frozen_spans(sc)):
|
|
pos = m.end() + 1
|
|
continue
|
|
|
|
def give_up() -> tuple[str, int]:
|
|
return materialize_todos(add_sentinel(text, m.start(), R_LITERAL)), close_brace + 1
|
|
|
|
fields: dict[str, tuple[int, int]] = {}
|
|
ok = True
|
|
for s, e in split_top_level(sc, open_brace + 1, close_brace):
|
|
item = text[s:e]
|
|
fm = re.match(r"\s*(\w+)\s*:\s*", item)
|
|
if fm:
|
|
fields[fm.group(1)] = (s + fm.end(), e)
|
|
elif re.fullmatch(r"\s*\w+\s*", item):
|
|
name = item.strip()
|
|
fields[name] = (s + item.index(name), s + item.index(name) + len(name))
|
|
else:
|
|
ok = False
|
|
needed = {"method", "uri", "headers", "extensions", "input"}
|
|
optional = {"credentials", "region", "service", "trailing_headers"}
|
|
if not ok or not needed <= set(fields) or set(fields) - needed - optional:
|
|
text, pos = give_up()
|
|
continue
|
|
creds = None
|
|
if "credentials" in fields and strip_span(text, fields["credentials"]) != "None":
|
|
creds = _parse_credentials(sc, fields["credentials"])
|
|
if creds is None:
|
|
text, pos = give_up()
|
|
continue
|
|
if "trailing_headers" in fields and strip_span(text, fields["trailing_headers"]) != "None":
|
|
text, pos = give_up()
|
|
continue
|
|
|
|
base_indent = indent_of(text, m.start())
|
|
chain_indent = base_indent + " "
|
|
|
|
def value(name: str) -> str:
|
|
s, e = fields[name]
|
|
return _reindent(text[s:e].strip(), indent_of(text, s), chain_indent)
|
|
|
|
parts = [f"crate::admin::operation::test_support::TestRequest::new({value('method')}, {_unwrap_uri(value('uri'))})"]
|
|
parts.append(f".headers({value('headers')})")
|
|
parts.append(f".extensions({value('extensions')})")
|
|
parts.append(f".body({value('input')})")
|
|
if creds is not None:
|
|
parts.append(f".credentials({creds[0]}, {creds[1]}, None)")
|
|
reasons: list[str] = []
|
|
for opt in ("region", "service"):
|
|
if opt in fields and strip_span(text, fields[opt]) != "None":
|
|
parts.append(f".{opt}({value(opt)})")
|
|
reasons.append(R_LITERAL_REGION)
|
|
parts.append(".build()")
|
|
replacement = parts[0] + "".join(f"\n{chain_indent}{p}" for p in parts[1:])
|
|
text = text[:m.start()] + replacement + text[close_brace + 1:]
|
|
for reason in reasons:
|
|
text = add_sentinel(text, m.start(), reason)
|
|
text = materialize_todos(text)
|
|
pos = m.start() + len(replacement)
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: types
|
|
|
|
|
|
def rename_types(text: str, ctx: Ctx) -> str:
|
|
text = simple_sub(text, r"(?<![\w:])(?:s3s::)?S3Request<\s*(?:s3s::)?Body\s*>", "AdminRequest", ctx=ctx)
|
|
text = simple_sub(
|
|
text,
|
|
r"(?<![\w:])(?:s3s::)?S3Response<\s*\(\s*(?:http::|hyper::)?StatusCode\s*,\s*(?:s3s::)?Body\s*\)\s*>",
|
|
"AdminResponse",
|
|
ctx=ctx,
|
|
)
|
|
text = rewrite(text, re.compile(r"(?<![\w:])(?:s3s::)?S3Response<"), lambda m, sc, scopes: (None, R_RESP_TYPE), ctx=ctx)
|
|
text = rewrite(text, re.compile(r"(?<![\w:])(?:s3s::)?S3Request<"), lambda m, sc, scopes: (None, R_REQ_TYPE), ctx=ctx)
|
|
sc = Scanner(text)
|
|
for f in reversed(iter_fns(sc)):
|
|
reason = ctx.frozen.get(f.key)
|
|
if reason:
|
|
text = add_sentinel(text, f.sig_start, reason)
|
|
return materialize_todos(text)
|
|
|
|
|
|
def qualify_frozen(text: str, ctx: Ctx) -> str:
|
|
"""Frozen functions keep s3s types; spell them out so the s3s imports can go."""
|
|
if not ctx.frozen:
|
|
return text
|
|
sc = Scanner(text)
|
|
spans = ctx.frozen_spans(sc)
|
|
out = text
|
|
for m in reversed(list(re.finditer(r"(?<![\w:])(S3Request|S3Response|Body)\b", text))):
|
|
if sc.is_code(m.start()) and in_spans(m.start(), spans):
|
|
out = out[:m.start()] + "s3s::" + out[m.start():]
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: helpers and fields
|
|
|
|
_HELPER_RENAMES = {
|
|
"authorize_admin_request": "authorize_admin_envelope",
|
|
"object_store_from_req": "object_store_from_envelope",
|
|
"app_context_from_req": "app_context_from_envelope",
|
|
}
|
|
|
|
|
|
def rewrite_helpers(text: str, ctx: Ctx) -> str:
|
|
text = simple_sub(text, r"\b(\w+)_from_extensions\(\s*&(\w+)\.extensions\b", r"\1_from_envelope(\2.envelope()", ctx=ctx)
|
|
text = req_sub(
|
|
text,
|
|
r"\bKmsAdminAudit::from_request\(\s*&(\w+)\.extensions\s*,\s*&\1\.headers\s*,\s*",
|
|
r"KmsAdminAudit::from_envelope(\1.envelope(), ",
|
|
None,
|
|
ctx,
|
|
)
|
|
text = req_sub(text, r"\bCallerIdentity::resolve\(\s*&?(\w+)\s*\)", r"CallerIdentity::resolve_envelope(\1.envelope())", None, ctx)
|
|
text = req_sub(text, r"\b(AccountAudit\w*)::from_request\(\s*&?(\w+)\s*\)", r"\1::from_envelope(\2.envelope())", None, ctx, group=2)
|
|
|
|
rx = re.compile(r"\bauthorize_admin_request\((\s*)&?(\w+)(\s*),")
|
|
|
|
def auth_fn(m, sc, scopes):
|
|
if sc.is_code(m.start()) and not is_request_ident(m.group(2), m.start(2), scopes):
|
|
return None
|
|
return (f"authorize_admin_envelope({m.group(1)}{m.group(2)}.envelope(){m.group(3)},", None)
|
|
|
|
# Also inside string literals: source-inspecting tests grep their own file.
|
|
text = rewrite(text, rx, auth_fn, ctx=ctx, only_code=False)
|
|
text = req_sub(text, r"\b(object_store|app_context)_from_req\(\s*&?(\w+)\s*\)", r"\1_from_envelope(\2.envelope())", None, ctx, group=2)
|
|
return text
|
|
|
|
|
|
SEP = r"(\s*\.\s*)" # `req.field` or `req\n .field`
|
|
|
|
|
|
def rewrite_fields(text: str, ctx: Ctx) -> str:
|
|
rx = re.compile(
|
|
r"\b(\w+)\s*\.\s*extensions\s*\.\s*get::<Option<RemoteAddr>>\(\)\s*"
|
|
r"(?:\.\s*and_then\(\|(\w+)\|\s*\2\.map\(\|(\w+)\|\s*\3\.0((?:\.\w+\(\))*)\)\)|\.\s*copied\(\)\s*\.\s*flatten\(\))"
|
|
)
|
|
|
|
def remote_fn(m, sc, scopes):
|
|
if not is_request_ident(m.group(1), m.start(1), scopes):
|
|
return None
|
|
tail = m.group(4) or ""
|
|
if tail:
|
|
return (f"{m.group(1)}.remote_addr().map(|{m.group(3)}| {m.group(3)}{tail})", None)
|
|
return (f"{m.group(1)}.remote_addr()", None)
|
|
|
|
text = rewrite(text, rx, remote_fn, ctx=ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}extensions\s*\.\s*get::<ReqInfo>\(\)", r"\1\2req_info()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}extensions\s*\.\s*get_mut::<ReqInfo>\(\)", r"\1\2req_info_mut()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}extensions\s*\.\s*insert\((?=ReqInfo\s*\{{|req_info\b)", r"\1\2install_req_info(", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}extensions\s*\.\s*get::<ClientInfo>\(\)", r"\1\2client_info()", None, ctx)
|
|
text = req_sub(
|
|
text,
|
|
rf"request_context:\s*(\w+){SEP}extensions\s*\.\s*get\(\)\s*\.\s*cloned\(\)",
|
|
r"request_context: \1\2request_context().cloned()",
|
|
None,
|
|
ctx,
|
|
)
|
|
text = req_sub(text, rf"\b(\w+){SEP}extensions\b", None, R_EXT, ctx)
|
|
|
|
# uri
|
|
text = req_sub(text, rf"\b(\w+){SEP}uri\s*=(?!=)", None, R_URI_ASSIGN, ctx)
|
|
text = req_sub(text, rf"&(\w+){SEP}uri\b(?!\s*\()", r"\1\2uri()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}uri\b(?![\s(.]|\s*=[^=])", r"\1\2uri()", R_BORROW, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}uri\b(?!\s*\()(?!\s*=[^=])", r"\1\2uri()", None, ctx)
|
|
# headers
|
|
text = req_sub(
|
|
text, rf"\b(\w+){SEP}headers\s*\.\s*(?:insert|append|remove|clear|entry|get_mut|extend|drain)\b", None, R_HDR_MUT, ctx
|
|
)
|
|
text = req_sub(text, rf"&(\w+){SEP}headers\b(?!\s*\()", r"\1\2headers()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}headers\b(?![\s(.]|\s*=[^=])", r"\1\2headers()", R_BORROW, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}headers\b(?!\s*\()(?!\s*=[^=])", r"\1\2headers()", None, ctx)
|
|
# method
|
|
text = req_sub(text, rf"\b(\w+){SEP}method\s*=(?!=)", None, R_METHOD_ASSIGN, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}method\b(?!\s*\()(?!\s*=[^=])", r"\1\2method()", None, ctx)
|
|
# region / service
|
|
text = req_sub(text, rf"\b(\w+){SEP}(?:region|service)\b(?!\s*\()", None, R_REGION, ctx)
|
|
# credentials
|
|
text = req_sub(text, rf"\b(\w+){SEP}credentials\s*=(?!=)", None, R_CRED_ASSIGN, ctx)
|
|
text = req_sub(
|
|
text,
|
|
rf"let\s+Some\(\s*ref\s+(\w+)\s*\)\s*=\s*(\w+){SEP}credentials\b(?!\s*\()",
|
|
r"let Some(\1) = \2\3credentials()",
|
|
None,
|
|
ctx,
|
|
group=2,
|
|
)
|
|
text = req_sub(text, rf"&(\w+){SEP}credentials\b(?!\s*\()", r"\1\2credentials()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}credentials\s*\.\s*as_ref\(\)", r"\1\2credentials()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}credentials\b(?![\s(.]|\s*=[^=])", r"\1\2credentials()", R_BORROW, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}credentials\b(?!\s*\()(?!\s*=[^=])", r"\1\2credentials()", None, ctx)
|
|
text = credential_fields(text, ctx)
|
|
return text
|
|
|
|
|
|
_CRED_BIND_RE = re.compile(r"Some\(\s*(\w+)\s*\)\s*=\s*(\w+)\s*\.\s*credentials\(\)")
|
|
|
|
|
|
def credential_fields(text: str, ctx: Ctx) -> str:
|
|
"""Flag (or with ``--credential-methods`` rewrite) field access on ``req.credentials()`` bindings."""
|
|
sc = Scanner(text)
|
|
fns = iter_fns(sc)
|
|
frozen = ctx.frozen_spans(sc, fns)
|
|
scopes = request_scopes(sc, fns)
|
|
edits: list[tuple[int, int, str | None, str | None]] = []
|
|
seen: set[int] = set()
|
|
for m in _CRED_BIND_RE.finditer(text):
|
|
if not sc.is_code(m.start()) or in_spans(m.start(), frozen):
|
|
continue
|
|
if not is_request_ident(m.group(2), m.start(2), scopes):
|
|
continue
|
|
name = m.group(1)
|
|
f = enclosing_fn(fns, m.start())
|
|
if f is None:
|
|
continue
|
|
region_start = line_end(text, m.end())
|
|
region_end = f.body[1]
|
|
esc = re.escape(name)
|
|
rebind = re.compile(rf"\blet\s+(?:mut\s+)?(?:\(\s*)?(?:\w+\s*,\s*)*{esc}\b|\blet\s+Some\(\s*(?:ref\s+)?{esc}\s*\)")
|
|
rb = sc.next_code(rebind, region_start)
|
|
if rb is not None and rb.start() < region_end:
|
|
# The shadowing statement's initializer may still read the old binding.
|
|
semi = sc.next_code(re.compile(";"), rb.end())
|
|
region_end = min(region_end, semi.end() if semi else rb.end())
|
|
field_rx = re.compile(rf"(&)?(?<![\w.]){esc}\.(access_key|secret_key|session_token)\b(\.(\w+)\(\))?(?!\s*\()")
|
|
pos = region_start
|
|
while True:
|
|
fm = sc.next_code(field_rx, pos)
|
|
if fm is None or fm.start() >= region_end:
|
|
break
|
|
pos = fm.end()
|
|
if fm.start() in seen:
|
|
continue
|
|
seen.add(fm.start())
|
|
amp, fld, call, callee = fm.group(1), fm.group(2), fm.group(3), fm.group(4)
|
|
repl = None
|
|
if ctx.credential_methods:
|
|
if call is None and (amp or fld == "session_token"):
|
|
repl = f"{name}.{fld}()"
|
|
elif callee in {"expose", "as_str", "as_deref"} and not amp:
|
|
repl = f"{name}.{fld}()"
|
|
elif callee == "clone" and not amp and fld != "session_token":
|
|
repl = f"{name}.{fld}().to_string()"
|
|
edits.append((fm.start(), fm.end(), repl, None if repl else R_CRED_FIELD))
|
|
out = text
|
|
for s, e, repl, reason in sorted(edits, reverse=True):
|
|
if repl is not None:
|
|
out = out[:s] + repl + out[e:]
|
|
if reason:
|
|
out = add_sentinel(out, s, reason)
|
|
return materialize_todos(out)
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: body
|
|
|
|
|
|
def rewrite_body(text: str, ctx: Ctx) -> str:
|
|
text = req_sub(text, r"std::mem::take\(\s*&mut\s+(\w+)\s*\.\s*input\s*\)", r"\1.take_body()", None, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}input\s*=(?!=)", None, R_INPUT_ASSIGN, ctx)
|
|
text = req_sub(text, rf"\b(\w+){SEP}input\b(?!\s*\()(?!\s*=[^=])", r"\1\2take_body()", None, ctx)
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: responses
|
|
|
|
_RESP_CTOR_RE = re.compile(r"(?<![\w:])(?:s3s::)?S3Response::(new|with_headers)\(")
|
|
|
|
|
|
def rewrite_responses(text: str, ctx: Ctx) -> str:
|
|
pos = 0
|
|
while True:
|
|
sc = Scanner(text)
|
|
m = sc.next_code(_RESP_CTOR_RE, pos)
|
|
if m is None:
|
|
break
|
|
if in_spans(m.start(), ctx.frozen_spans(sc)):
|
|
pos = m.end()
|
|
continue
|
|
kind = m.group(1)
|
|
p_open = m.end() - 1
|
|
p_close = sc.match(p_open)
|
|
if p_close is None:
|
|
pos = m.end()
|
|
continue
|
|
args = split_top_level(sc, p_open + 1, p_close)
|
|
expected = 1 if kind == "new" else 2
|
|
elems: list[str] = []
|
|
if len(args) == expected:
|
|
s, e = args[0]
|
|
first = text[s:e]
|
|
t_open = s + (len(first) - len(first.lstrip()))
|
|
t_close = sc.match(t_open) if text[t_open] == "(" else None
|
|
if t_close is not None and not text[t_close + 1:e].strip():
|
|
inner = split_top_level(sc, t_open + 1, t_close)
|
|
if len(inner) == 2:
|
|
elems = [strip_span(text, inner[0]), strip_span(text, inner[1])]
|
|
if not elems:
|
|
text = materialize_todos(add_sentinel(text, m.start(), R_RESP_CTOR))
|
|
pos = p_close + 1
|
|
continue
|
|
if kind == "with_headers":
|
|
elems.append(strip_span(text, args[1]))
|
|
close_ls = line_start(text, p_close)
|
|
if "\n" in text[m.start():p_close + 1] and not text[close_ls:p_close].strip():
|
|
close_indent = text[close_ls:p_close]
|
|
elem_indent = close_indent + " "
|
|
body = "".join(f"\n{elem_indent}{el}," for el in elems)
|
|
replacement = f"AdminResponse::{kind}({body}\n{close_indent})"
|
|
else:
|
|
replacement = f"AdminResponse::{kind}({', '.join(elems)})"
|
|
text = text[:m.start()] + replacement + text[p_close + 1:]
|
|
pos = m.start() + len(replacement)
|
|
text = simple_sub(text, r"(?<![\w:])s3s::Body::", "Body::", ctx=ctx)
|
|
|
|
# Field access on owned responses.
|
|
sc = Scanner(text)
|
|
fns = iter_fns(sc)
|
|
bound: list[tuple[str, int, int]] = []
|
|
for m in re.finditer(r"\blet\s+(?:mut\s+)?(\w+)\s*=\s*AdminResponse::", text):
|
|
if not sc.is_code(m.start()):
|
|
continue
|
|
f = enclosing_fn(fns, m.start())
|
|
bound.append((m.group(1), m.end(), f.body[1] if f else len(text)))
|
|
|
|
def resp_field(m, sc, scopes):
|
|
if any(n == m.group(1) and s <= m.start() < e for n, s, e in bound):
|
|
return (None, R_RESP_FIELD)
|
|
return None
|
|
|
|
text = rewrite(text, re.compile(r"\b(\w+)\.(?:headers|status|body|extensions|output)\b"), resp_field, ctx=ctx)
|
|
frozen_names = {name for name, _ in ctx.frozen}
|
|
|
|
def output_fn(m, sc, scopes):
|
|
name = m.group(1)
|
|
if name == "self":
|
|
return None
|
|
f = enclosing_fn(fns, m.start())
|
|
if f is not None and frozen_names:
|
|
body_before = text[f.body[0]:m.start()]
|
|
binding = None
|
|
for bm in re.finditer(rf"\blet\s+(?:mut\s+)?{re.escape(name)}\s*=([^;]*);", body_before):
|
|
binding = bm.group(1)
|
|
if binding and any(re.search(rf"\b{re.escape(fz)}\s*\(", binding) for fz in frozen_names):
|
|
return None
|
|
return (None, R_OUTPUT)
|
|
|
|
text = rewrite(text, re.compile(r"(?<![\w.])(\w+)\.output\b"), output_fn, ctx=ctx)
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: mut params
|
|
|
|
_MUT_NEED_TMPL = (
|
|
r"\b{name}\s*\.\s*(?:take_body|req_info_mut|install_req_info|envelope_mut)\(|&mut\s+{name}\b|\b{name}\s*\.\s*\w+\s*=[^=]"
|
|
r"|\b{name}\s*\.\s*headers\(\)\s*\.\s*(?:insert|append|remove|clear|entry|get_mut|extend|drain)\("
|
|
)
|
|
|
|
|
|
def normalize_mut(text: str, ctx: Ctx) -> str:
|
|
sc = Scanner(text)
|
|
fns = iter_fns(sc)
|
|
frozen = ctx.frozen_spans(sc, fns)
|
|
edits: list[tuple[int, int, str]] = []
|
|
for f in fns:
|
|
if in_spans(f.sig_start, frozen):
|
|
continue
|
|
params = text[f.params[0]:f.params[1]]
|
|
body = text[f.body[0]:f.body[1]]
|
|
for m in re.finditer(r"(?<![\w&])(mut\s+)?(\w+)\s*:\s*AdminRequest\b", params):
|
|
if params[:m.start()].rstrip().endswith("&"):
|
|
continue
|
|
name = m.group(2)
|
|
needs = re.search(_MUT_NEED_TMPL.format(name=re.escape(name)), body) is not None
|
|
# `let mut req = req;` rebinds the parameter; the parameter itself
|
|
# must then stay immutable or rustc reports `unused_mut`.
|
|
if re.search(rf"let\s+mut\s+{re.escape(name)}\s*=\s*{re.escape(name)}\s*;", body):
|
|
needs = False
|
|
has = m.group(1) is not None
|
|
abs_start = f.params[0] + m.start()
|
|
if needs and not has:
|
|
edits.append((abs_start, abs_start, "mut "))
|
|
elif has and not needs:
|
|
edits.append((abs_start, abs_start + len(m.group(1)), ""))
|
|
out = text
|
|
for s, e, repl in sorted(edits, reverse=True):
|
|
out = out[:s] + repl + out[e:]
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------- use-tree model
|
|
|
|
|
|
@dataclass
|
|
class UseTree:
|
|
path: list[str]
|
|
children: list["UseTree"] | None = None
|
|
alias: str | None = None
|
|
|
|
@property
|
|
def leaf(self) -> str:
|
|
return self.path[-1] if self.path else ""
|
|
|
|
|
|
def parse_use_tree(s: str) -> UseTree:
|
|
s = s.strip()
|
|
pos = 0
|
|
|
|
def ws(pattern: str) -> re.Match[str] | None:
|
|
nonlocal pos
|
|
m = re.match(pattern, s[pos:])
|
|
if m:
|
|
pos += m.end()
|
|
return m
|
|
|
|
def parse_group() -> list[UseTree]:
|
|
children: list[UseTree] = []
|
|
while not ws(r"\s*\}"):
|
|
children.append(parse_node())
|
|
ws(r"\s*,")
|
|
return children
|
|
|
|
def parse_node() -> UseTree:
|
|
path: list[str] = []
|
|
while True:
|
|
m = ws(r"\s*(\w+|\*)")
|
|
if m:
|
|
path.append(m.group(1))
|
|
if ws(r"\s*::"):
|
|
if ws(r"\s*\{"):
|
|
return UseTree(path, parse_group())
|
|
continue
|
|
am = ws(r"\s*as\s+(\w+)")
|
|
return UseTree(path, None, am.group(1) if am else None)
|
|
if ws(r"\s*\{"):
|
|
return UseTree(path, parse_group())
|
|
raise ValueError(f"cannot parse use tree: {s!r} at {pos}")
|
|
|
|
return parse_node()
|
|
|
|
|
|
def _version_key(seg: str) -> tuple:
|
|
if seg == "self":
|
|
return (0,)
|
|
if seg == "super":
|
|
return (1,)
|
|
if seg == "crate":
|
|
return (2,)
|
|
parts = re.findall(r"\d+|\D+", seg)
|
|
return (3,) + tuple((1, int(p)) if p.isdigit() else (0, p) for p in parts)
|
|
|
|
|
|
def tree_sort_key(t: UseTree) -> tuple:
|
|
key = tuple(_version_key(p) for p in t.path)
|
|
if t.children is not None:
|
|
key = key + ((9,),)
|
|
return key
|
|
|
|
|
|
def serialize_tree(t: UseTree) -> str:
|
|
base = "::".join(t.path)
|
|
if t.children is None:
|
|
return base + (f" as {t.alias}" if t.alias else "")
|
|
kids = ", ".join(serialize_tree(c) for c in sorted(t.children, key=tree_sort_key))
|
|
return f"{base}::{{{kids}}}" if base else f"{{{kids}}}"
|
|
|
|
|
|
def _format_tree(t: UseTree, indent: str, prefix_len: int, suffix_len: int) -> str:
|
|
"""rustfmt-like layout: one line if it fits, vertical if nested groups, else fill."""
|
|
one = serialize_tree(t)
|
|
if t.children is None:
|
|
return one
|
|
kids = sorted(t.children, key=tree_sort_key)
|
|
nested = any(c.children is not None for c in kids)
|
|
if not nested and len(indent) + prefix_len + len(one) + suffix_len <= MAX_WIDTH:
|
|
return one
|
|
base = "::".join(t.path)
|
|
inner = indent + " "
|
|
head = f"{base}::{{" if base else "{"
|
|
if nested:
|
|
lines = [head]
|
|
for c in kids:
|
|
lines.append(inner + _format_tree(c, inner, 0, 1) + ",")
|
|
lines.append(indent + "}")
|
|
return "\n".join(lines)
|
|
lines = [head]
|
|
cur = ""
|
|
for c in kids:
|
|
piece = serialize_tree(c) + ","
|
|
if cur and len(inner) + len(cur) + 1 + len(piece) > MAX_WIDTH:
|
|
lines.append(inner + cur)
|
|
cur = piece
|
|
else:
|
|
cur = f"{cur} {piece}" if cur else piece
|
|
lines.append(inner + cur)
|
|
lines.append(indent + "}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def serialize_use(t: UseTree, indent: str, attrs: str = "") -> str:
|
|
return f"{attrs}{indent}use {_format_tree(t, indent, 4, 1)};"
|
|
|
|
|
|
def collapse(t: UseTree) -> UseTree:
|
|
"""Fold ``a::{b}`` into ``a::b`` and ``a::{self}`` into ``a``."""
|
|
if t.children is None:
|
|
return t
|
|
t.children = [collapse(c) for c in t.children]
|
|
if len(t.children) == 1:
|
|
c = t.children[0]
|
|
if c.path == ["self"] and c.children is None:
|
|
return UseTree(t.path, None, c.alias)
|
|
if c.path and c.path[0] != "self":
|
|
return UseTree(t.path + c.path, c.children, c.alias)
|
|
return t
|
|
|
|
|
|
def leaves(t: UseTree, prefix: list[str] | None = None) -> list[tuple[list[str], UseTree]]:
|
|
"""All leaves as (full path, node)."""
|
|
full = (prefix or []) + t.path
|
|
if t.children is None:
|
|
return [(full, t)]
|
|
out: list[tuple[list[str], UseTree]] = []
|
|
for c in t.children:
|
|
out.extend(leaves(c, full))
|
|
return out
|
|
|
|
|
|
@dataclass
|
|
class UseStmt:
|
|
start: int # includes attribute lines directly above
|
|
end: int # just after ';'
|
|
indent: str
|
|
attrs: str
|
|
tree: UseTree
|
|
top_level: bool
|
|
|
|
|
|
_USE_RE = re.compile(r"^([ \t]*)(pub(?:\([^)]*\))?\s+)?use\s", re.M)
|
|
|
|
|
|
def find_use_stmts(text: str, sc: Scanner | None = None) -> list[UseStmt]:
|
|
sc = sc or Scanner(text)
|
|
out: list[UseStmt] = []
|
|
for m in _USE_RE.finditer(text):
|
|
if not sc.is_code(m.start() + len(m.group(1))):
|
|
continue
|
|
if m.group(2):
|
|
continue # `pub use` re-exports are left alone
|
|
i = m.end()
|
|
while i < len(text) and not (sc.is_code(i) and text[i] == ";"):
|
|
i += 1
|
|
if i >= len(text):
|
|
continue
|
|
try:
|
|
tree = parse_use_tree(text[m.end():i])
|
|
except ValueError:
|
|
continue
|
|
start = m.start()
|
|
attrs = ""
|
|
while start > 0:
|
|
pls = line_start(text, start - 1)
|
|
prev = text[pls:start - 1]
|
|
if re.match(r"\s*#\[", prev):
|
|
attrs = prev + "\n" + attrs
|
|
start = pls
|
|
continue
|
|
break
|
|
out.append(UseStmt(start, i + 1, m.group(1), attrs, tree, m.group(1) == ""))
|
|
return out
|
|
|
|
|
|
def replace_stmt(text: str, stmt: UseStmt, new_tree: UseTree | None) -> str:
|
|
end = stmt.end
|
|
if new_tree is None:
|
|
if end < len(text) and text[end] == "\n":
|
|
end += 1
|
|
return text[:stmt.start] + text[end:]
|
|
return text[:stmt.start] + serialize_use(new_tree, stmt.indent, stmt.attrs) + text[end:]
|
|
|
|
|
|
def remove_leaves(tree: UseTree, pred) -> UseTree | None:
|
|
"""Drop leaves where ``pred(full_path, node)`` holds; prune/collapse; None when nothing is left."""
|
|
|
|
def walk(node: UseTree, prefix: list[str]) -> UseTree | None:
|
|
full = prefix + node.path
|
|
if node.children is None:
|
|
return None if pred(full, node) else node
|
|
kids = [k for k in (walk(c, full) for c in node.children) if k is not None]
|
|
node.children = kids
|
|
return node if kids else None
|
|
|
|
out = walk(tree, [])
|
|
return collapse(out) if out is not None else None
|
|
|
|
|
|
def _header_tree(names: list[str]) -> UseTree:
|
|
return collapse(UseTree(["http", "header"], [UseTree([n]) for n in sorted(dict.fromkeys(names), key=_version_key)]))
|
|
|
|
|
|
def _enclosing_block(sc: Scanner, pos: int) -> tuple[int, int]:
|
|
"""Span of the innermost ``{ .. }`` containing ``pos`` (whole file when none)."""
|
|
best = (0, len(sc.text))
|
|
for o, c in sc.pairs.items():
|
|
if o < c and sc.text[o] == "{" and o < pos < c and (c - o) < (best[1] - best[0]):
|
|
best = (o, c)
|
|
return best
|
|
|
|
|
|
def edit_use_trees(text: str, editor) -> str:
|
|
"""Edit leaves in place.
|
|
|
|
``editor(stmt, full_path, node) -> None`` keeps the leaf, ``[]`` drops it and a
|
|
list of nodes replaces it (paths relative to the leaf's parent group). Empty
|
|
groups are pruned and single-item groups collapsed afterwards.
|
|
"""
|
|
for stmt in reversed(find_use_stmts(text)):
|
|
changed = False
|
|
|
|
def walk(node: UseTree, prefix: list[str], is_root: bool) -> list[UseTree]:
|
|
nonlocal changed
|
|
full = prefix + node.path
|
|
if node.children is None:
|
|
res = editor(stmt, full, node)
|
|
if res is None:
|
|
return [node]
|
|
changed = True
|
|
if is_root and len(res) != 1:
|
|
return [UseTree(node.path[:-1], res)] if res else []
|
|
return res
|
|
kids: list[UseTree] = []
|
|
for c in node.children:
|
|
kids.extend(walk(c, full, False))
|
|
node.children = kids
|
|
return [node] if kids else []
|
|
|
|
new = walk(stmt.tree, [], True)
|
|
if not changed:
|
|
continue
|
|
text = replace_stmt(text, stmt, collapse(new[0]) if new else None)
|
|
return text
|
|
|
|
|
|
def code_uses(text: str, name: str, sc: Scanner | None = None, stmts: list[UseStmt] | None = None) -> bool:
|
|
"""True if ``name`` occurs unqualified in code outside ``use`` statements."""
|
|
sc = sc or Scanner(text)
|
|
stmts = stmts if stmts is not None else find_use_stmts(text, sc)
|
|
for m in re.finditer(rf"(?<![\w:]){re.escape(name)}\b", text):
|
|
if sc.is_code(m.start()) and not any(s.start <= m.start() < s.end for s in stmts):
|
|
return True
|
|
return False
|
|
|
|
|
|
def imported_names(stmts: list[UseStmt], *, top_level_only: bool = False) -> set[str]:
|
|
names: set[str] = set()
|
|
for s in stmts:
|
|
if top_level_only and not s.top_level:
|
|
continue
|
|
for path, node in leaves(s.tree):
|
|
names.add(node.alias or path[-1])
|
|
return names
|
|
|
|
|
|
def insert_top_level_use(text: str, new_tree: UseTree) -> str:
|
|
"""Insert a top-level ``use`` statement at its rustfmt-sorted position."""
|
|
stmts = [s for s in find_use_stmts(text) if s.top_level]
|
|
line = serialize_use(new_tree, "")
|
|
if not stmts:
|
|
m = re.search(r"^(?!//|#!|\s*$)", text, re.M)
|
|
pos = m.start() if m else 0
|
|
return text[:pos] + line + "\n\n" + text[pos:]
|
|
block = [stmts[0]]
|
|
for s in stmts[1:]:
|
|
gap = text[block[-1].end:s.start]
|
|
if gap.strip() == "" and gap.count("\n") <= 1:
|
|
block.append(s)
|
|
else:
|
|
break
|
|
new_key = tree_sort_key(new_tree)
|
|
for s in block:
|
|
if tree_sort_key(s.tree) > new_key:
|
|
return text[:s.start] + line + "\n" + text[s.start:]
|
|
last = block[-1]
|
|
return text[:last.end] + "\n" + line + text[last.end:]
|
|
|
|
|
|
def _insert_nested_use(text: str, stmt: UseStmt, tree: UseTree) -> str:
|
|
"""Insert ``tree`` as a use statement where the (nested) ``stmt`` used to be."""
|
|
return text[:stmt.start] + serialize_use(tree, stmt.indent) + "\n" + text[stmt.start:]
|
|
|
|
|
|
def merge_into_top_level(text: str, root: list[str], names: list[str]) -> str:
|
|
"""Add ``root::name`` leaves to an existing top-level statement rooted at ``root`` or insert one."""
|
|
stmts = [s for s in find_use_stmts(text) if s.top_level and s.tree.path[:len(root)] == root]
|
|
new_nodes = [UseTree([n]) for n in names]
|
|
if stmts:
|
|
stmt = stmts[0]
|
|
t = stmt.tree
|
|
if t.path == root and t.children is not None:
|
|
t.children.extend(new_nodes)
|
|
return replace_stmt(text, stmt, t)
|
|
rest = UseTree(t.path[len(root):], t.children, t.alias)
|
|
return replace_stmt(text, stmt, UseTree(root, [rest] + new_nodes))
|
|
return insert_top_level_use(text, collapse(UseTree(root, new_nodes)))
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: imports
|
|
|
|
_DROP_FROM_S3S = {"Body", "S3Request", "S3Response"}
|
|
|
|
|
|
def rewrite_imports(text: str, ctx: Ctx) -> str:
|
|
# LegacyAdminAdapter statics: `static X: LegacyAdminAdapter = LegacyAdminAdapter(&HANDLER);`
|
|
rx = re.compile(r"^[ \t]*static\s+(\w+)\s*:\s*LegacyAdminAdapter\s*=\s*LegacyAdminAdapter\(\s*&(\w+)\s*\)\s*;[ \t]*\n", re.M)
|
|
for m in reversed(list(rx.finditer(text))):
|
|
ctx.adapter_map[m.group(1)] = m.group(2)
|
|
text = text[:m.start()] + text[m.end():]
|
|
for adapter, handler in ctx.adapter_map.items():
|
|
text = simple_sub(text, rf"&{re.escape(adapter)}\b", f"&{handler}")
|
|
|
|
# router::Operation / router::LegacyAdminAdapter
|
|
def router_editor(stmt, full, node):
|
|
if full[-2:] == ["router", "LegacyAdminAdapter"]:
|
|
return []
|
|
if full[-2:] == ["router", "Operation"] and node.alias is None:
|
|
if len(node.path) >= 2:
|
|
# `use crate::admin::router::Operation;` keeps its slot (test modules rely on it).
|
|
return [UseTree(node.path[:-2] + ["operation", "Operation"])]
|
|
return [] # group member: re-imported from crate::admin::operation below
|
|
return None
|
|
|
|
text = edit_use_trees(text, router_editor)
|
|
|
|
# s3s imports: drop Body/S3Request/S3Response, move header::* to http::header
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
pending_top: list[list[str]] = []
|
|
for stmt in reversed(stmts):
|
|
if stmt.tree.path[:1] != ["s3s"]:
|
|
continue
|
|
headers: list[str] = []
|
|
dropped = False
|
|
for full, node in leaves(stmt.tree):
|
|
if node.alias is not None:
|
|
continue
|
|
if full[:2] == ["s3s", "header"] and len(full) == 3:
|
|
headers.append(full[2])
|
|
elif len(full) == 2 and full[1] in _DROP_FROM_S3S:
|
|
dropped = True
|
|
if not headers and not dropped:
|
|
continue
|
|
|
|
def gone(full, node):
|
|
if node.alias is not None:
|
|
return False
|
|
return (full[:2] == ["s3s", "header"] and len(full) == 3) or (len(full) == 2 and full[1] in _DROP_FROM_S3S)
|
|
|
|
new_tree = remove_leaves(stmt.tree, gone)
|
|
if stmt.top_level:
|
|
text = replace_stmt(text, stmt, new_tree)
|
|
if headers:
|
|
pending_top.append(headers)
|
|
continue
|
|
# Nested module: the http import takes the s3s import's slot, deduplicated within the block.
|
|
block = _enclosing_block(sc, stmt.start)
|
|
scope = {
|
|
path[-1]
|
|
for s2 in stmts
|
|
if block[0] <= s2.start < block[1]
|
|
for path, _ in leaves(s2.tree)
|
|
if path[:2] in (["http", "header"], ["hyper", "header"])
|
|
}
|
|
names = [h for h in dict.fromkeys(headers) if h not in scope]
|
|
pieces = []
|
|
if names:
|
|
pieces.append(serialize_use(_header_tree(names), stmt.indent))
|
|
if new_tree is not None:
|
|
pieces.append(serialize_use(new_tree, stmt.indent, stmt.attrs))
|
|
if pieces:
|
|
text = text[:stmt.start] + "\n".join(pieces) + text[stmt.end:]
|
|
else:
|
|
text = replace_stmt(text, stmt, None)
|
|
for headers in pending_top:
|
|
top_stmts = [s2 for s2 in find_use_stmts(text) if s2.top_level]
|
|
already = {path[-1] for s2 in top_stmts for path, _ in leaves(s2.tree) if path[:2] in (["http", "header"], ["hyper", "header"])}
|
|
names = [h for h in dict.fromkeys(headers) if h not in already]
|
|
if names:
|
|
text = insert_top_level_use(text, _header_tree(names))
|
|
|
|
# crate::admin::storage_api::s3 facade imports
|
|
def storage_editor(stmt, full, node):
|
|
if full[:4] == ["crate", "admin", "storage_api", "s3"] and len(full) == 5 and full[4] in _DROP_FROM_S3S and node.alias is None:
|
|
return []
|
|
return None
|
|
|
|
text = edit_use_trees(text, storage_editor)
|
|
|
|
# helper renames in imports
|
|
renames = dict(_HELPER_RENAMES)
|
|
for m in re.finditer(r"\b(\w+)_from_envelope\b", text):
|
|
renames[f"{m.group(1)}_from_extensions"] = f"{m.group(1)}_from_envelope"
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
imported = imported_names(stmts)
|
|
for old, new in renames.items():
|
|
if old not in imported or not code_uses(text, new, sc, stmts):
|
|
continue
|
|
old_used = code_uses(text, old, sc, stmts)
|
|
new_imported = new in imported
|
|
|
|
def ren(stmt, full, node, old=old, new=new, old_used=old_used, new_imported=new_imported):
|
|
if full[-1] != old or node.alias is not None:
|
|
return None
|
|
renamed = UseTree(node.path[:-1] + [new])
|
|
if old_used:
|
|
return [node] if new_imported else [node, renamed]
|
|
return [renamed] if not new_imported else []
|
|
|
|
text = edit_use_trees(text, ren)
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
imported = imported_names(stmts)
|
|
|
|
# crate::admin::operation imports
|
|
needed = [n for n in ("AdminRequest", "AdminResponse") if code_uses(text, n, sc, stmts)]
|
|
if re.search(r"^impl Operation for\b", text, re.M):
|
|
needed.append("Operation")
|
|
if needed:
|
|
op_imported = {path[-1] for s in stmts if s.top_level for path, _ in leaves(s.tree) if path[:3] == ["crate", "admin", "operation"]}
|
|
missing = [n for n in needed if n not in op_imported]
|
|
if missing:
|
|
text = merge_into_top_level(text, ["crate", "admin", "operation"], missing)
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
|
|
# Body from rustfs_s3_types
|
|
if code_uses(text, "Body", sc, stmts) and "Body" not in imported_names(stmts, top_level_only=True):
|
|
text = merge_into_top_level(text, ["rustfs_s3_types"], ["Body"])
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: prune unused imports
|
|
|
|
PRUNE_NAMES = ["RemoteAddr", "ReqInfo", "ClientInfo", "Uri", "LegacyAdminAdapter", "S3Request", "S3Response", "Body"]
|
|
|
|
|
|
def prune_unused_imports(text: str) -> str:
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
imported = imported_names(stmts)
|
|
for name in PRUNE_NAMES:
|
|
if name not in imported or code_uses(text, name, sc, stmts):
|
|
continue
|
|
text = edit_use_trees(text, lambda stmt, full, node, name=name: [] if full[-1] == name and node.alias is None else None)
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: nested module imports
|
|
|
|
_MOD_RE = re.compile(r"^([ \t]*)(?:pub(?:\([^)]*\))?\s+)?mod\s+\w+\s*\{", re.M)
|
|
_NESTED_NAMES = ("AdminRequest", "AdminResponse", "Operation", "Body")
|
|
|
|
|
|
def fix_nested_imports(text: str) -> str:
|
|
"""Inline modules that import explicitly (no ``use super::*``) need the owned names themselves."""
|
|
sc = Scanner(text)
|
|
stmts = find_use_stmts(text, sc)
|
|
edits: list[tuple[int, int, str]] = []
|
|
for m in _MOD_RE.finditer(text):
|
|
if not sc.is_code(m.start()):
|
|
continue
|
|
open_b = m.end() - 1
|
|
close_b = sc.match(open_b)
|
|
if close_b is None:
|
|
continue
|
|
block_stmts = [st for st in stmts if open_b < st.start < close_b]
|
|
if any(path == ["super", "*"] for st in block_stmts for path, _ in leaves(st.tree)):
|
|
continue
|
|
imported = {node.alias or path[-1] for st in block_stmts for path, node in leaves(st.tree)}
|
|
needed = []
|
|
for name in _NESTED_NAMES:
|
|
if name in imported:
|
|
continue
|
|
for um in re.finditer(rf"(?<![\w:]){re.escape(name)}\b", text[open_b:close_b]):
|
|
pos = open_b + um.start()
|
|
if sc.is_code(pos) and not any(st.start <= pos < st.end for st in block_stmts):
|
|
needed.append(name)
|
|
break
|
|
if not needed:
|
|
continue
|
|
indent = m.group(1) + " "
|
|
ops = [n for n in needed if n != "Body"]
|
|
lines = []
|
|
if ops:
|
|
lines.append(serialize_use(collapse(UseTree(["crate", "admin", "operation"], [UseTree([n]) for n in ops])), indent))
|
|
if "Body" in needed:
|
|
lines.append(serialize_use(UseTree(["rustfs_s3_types", "Body"]), indent))
|
|
anchor = block_stmts[0].start if block_stmts else line_end(text, open_b) + 1
|
|
edits.append((anchor, anchor, "\n".join(lines) + "\n"))
|
|
for s_, e_, repl in sorted(edits, reverse=True):
|
|
text = text[:s_] + repl + text[e_:]
|
|
return text
|
|
|
|
|
|
# --------------------------------------------------------------------------- phase: strings
|
|
|
|
_STRING_SYMBOL_RE = re.compile(
|
|
r"S3Request\b|S3Response\b|authorize_admin_request|_from_req\(|_from_extensions\("
|
|
r"|req\.(?:extensions|credentials|input|headers|uri)\b(?!\s*\()"
|
|
)
|
|
|
|
# Source-inspecting tests assert on code fragments; keep those fragments in step with the code rewrites.
|
|
_STRING_MIRROR = [
|
|
(re.compile(r"authorize_admin_request\((\s*)&?req(\s*),"), r"authorize_admin_envelope(\1req.envelope()\2,"),
|
|
(re.compile(r"authorize_admin_request\("), "authorize_admin_envelope("),
|
|
(re.compile(r"\b(\w+)_from_extensions\(&req\.extensions"), r"\1_from_envelope(req.envelope()"),
|
|
(re.compile(r"\b(object_store|app_context)_from_req\(&?req\)"), r"\1_from_envelope(req.envelope())"),
|
|
(re.compile(r"KmsAdminAudit::from_request\(&req\.extensions, &req\.headers, "), "KmsAdminAudit::from_envelope(req.envelope(), "),
|
|
(re.compile(r"CallerIdentity::resolve\(&req\)"), "CallerIdentity::resolve_envelope(req.envelope())"),
|
|
(re.compile(r"std::mem::take\(&mut req\.input\)"), "req.take_body()"),
|
|
(re.compile(r"&req\.(uri|headers)\b"), r"req.\1()"),
|
|
(re.compile(r"\breq\.(uri|headers|method)\."), r"req.\1()."),
|
|
(re.compile(r"\breq\.credentials\.(is_none|as_ref)\(\)"), r"req.credentials().\1()"),
|
|
(re.compile(r"&req\.credentials\b"), "req.credentials()"),
|
|
(re.compile(r"\breq\.input\b(?!\s*=)"), "req.take_body()"),
|
|
(re.compile(r"(?<![\w:])S3Request<Body>"), "AdminRequest"),
|
|
(re.compile(r"(?<![\w:])S3Response<\(StatusCode, Body\)>"), "AdminResponse"),
|
|
]
|
|
|
|
|
|
def rewrite_strings(text: str, ctx: Ctx) -> str:
|
|
sc = Scanner(text)
|
|
for s, e in reversed(sc.strings):
|
|
lit = text[s:e]
|
|
new = lit
|
|
for rx, repl in _STRING_MIRROR:
|
|
new = rx.sub(repl, new)
|
|
if new != lit:
|
|
text = text[:s] + new + text[e:]
|
|
sc = Scanner(text)
|
|
for s, e in reversed(sc.strings):
|
|
if _STRING_SYMBOL_RE.search(text[s:e]):
|
|
text = add_sentinel(text, s, R_STRING)
|
|
return materialize_todos(text)
|
|
|
|
|
|
# --------------------------------------------------------------------------- driver
|
|
|
|
_NEEDS_RE = re.compile(r"\bS3Request\b|\bS3Response\b|\bimpl Operation for\b|router::Operation\b|\bLegacyAdminAdapter\b")
|
|
|
|
|
|
def transform(text: str, ctx: Ctx) -> str:
|
|
if not _NEEDS_RE.search(text):
|
|
return text
|
|
ctx.frozen = detect_frozen_fns(text, ctx.path)
|
|
text = convert_request_literals(text, ctx)
|
|
text = rename_types(text, ctx)
|
|
text = qualify_frozen(text, ctx)
|
|
text = rewrite_helpers(text, ctx)
|
|
text = rewrite_fields(text, ctx)
|
|
text = rewrite_body(text, ctx)
|
|
text = rewrite_responses(text, ctx)
|
|
text = normalize_mut(text, ctx)
|
|
text = rewrite_imports(text, ctx)
|
|
text = rewrite_strings(text, ctx)
|
|
text = prune_unused_imports(text)
|
|
text = fix_nested_imports(text)
|
|
return text
|
|
|
|
|
|
def todo_list(text: str) -> list[tuple[int, str]]:
|
|
out = []
|
|
for i, line in enumerate(text.split("\n"), 1):
|
|
s = line.strip()
|
|
if s.startswith(TODO_PREFIX.strip()):
|
|
out.append((i, s[len(TODO_PREFIX.strip()):].strip()))
|
|
return out
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--check", action="store_true", help="report changes and TODOs without writing")
|
|
ap.add_argument(
|
|
"--credential-methods",
|
|
action="store_true",
|
|
help="rewrite obvious cred.access_key/secret_key field access into accessor calls instead of only flagging it",
|
|
)
|
|
ap.add_argument("--rustfmt", action="store_true", help="run `rustfmt --edition 2024` on every written file")
|
|
ap.add_argument("files", nargs="+", type=Path)
|
|
args = ap.parse_args(argv)
|
|
|
|
total_files = 0
|
|
changed_files = 0
|
|
reasons: Counter[str] = Counter()
|
|
todo_total = 0
|
|
written: list[Path] = []
|
|
for path in args.files:
|
|
total_files += 1
|
|
original = path.read_text(encoding="utf-8")
|
|
ctx = Ctx(path=path, credential_methods=args.credential_methods)
|
|
result = transform(original, ctx)
|
|
todos = todo_list(result)
|
|
todo_total += len(todos)
|
|
for _, reason in todos:
|
|
reasons[reason] += 1
|
|
if result != original:
|
|
changed_files += 1
|
|
diff = list(difflib.unified_diff(original.split("\n"), result.split("\n"), lineterm="", n=0))
|
|
added = sum(1 for l in diff if l.startswith("+") and not l.startswith("+++"))
|
|
removed = sum(1 for l in diff if l.startswith("-") and not l.startswith("---"))
|
|
verb = "would change" if args.check else "changed"
|
|
print(f"{path}: {verb} (+{added} -{removed} lines), todos={len(todos)}")
|
|
if not args.check:
|
|
path.write_text(result, encoding="utf-8")
|
|
written.append(path)
|
|
else:
|
|
print(f"{path}: unchanged, todos={len(todos)}")
|
|
for line, reason in todos:
|
|
print(f" {path}:{line}: {reason}")
|
|
if args.rustfmt and written:
|
|
subprocess.run(["rustfmt", "--edition", "2024", *map(str, written)], check=False)
|
|
print(f"\nSummary: files={total_files} changed={changed_files} unchanged={total_files - changed_files} todos={todo_total}")
|
|
if reasons:
|
|
print("TODOs by reason:")
|
|
for reason, count in reasons.most_common():
|
|
print(f" {count:4d} {reason}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|