diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index 28231131c..526efc90c 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -27,7 +27,7 @@ jobs: include: - { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" } - { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" } - - { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "tofu-examples" } + - { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" } runs-on: [ self-hosted, "${{ matrix.runner }}" ] timeout-minutes: 35 diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 08b01bdc2..370c1ee63 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -2783,11 +2783,21 @@ func (d *Daemon) wireLBAgentConfig() { // Resolve gateway URL — the address LB VMs use to reach the AWS gateway. // Precedence: - // 1. AWSGW listens on a specific WAN IP + br-mgmt present → use WAN IP - // and add host route via br-mgmt for internal LBs. - // 2. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours + // 1. br-mgmt present + AWSGW on a dedicated IP distinct from AdvertiseIP + // (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach it) → + // gateway URL is the AWSGW bind IP and lb-agent gets a bootcmd host + // route via br-mgmt. + // 2. AdvertiseIP set (single-node, or multi-node where AWSGW binds to + // the advertised IP) → AdvertiseIP. VMs reach it via VPC → external + // (OVN's own dnat_and_snat SNATs their reply back to the ALB EIP). + // Critically, we do NOT add the mgmt host route here: when host IPs + // on the WAN share the advertiseIP, the /32 route would steal the + // return path for host-initiated ALB connections — replies would + // egress via mgmt with the VM's 10.x source IP, bypass OVN's SNAT, + // and arrive at the host with a source that doesn't match the open + // TCP socket (the client dialed the EIP, not the VM IP). + // 3. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours // reach the daemon via mgmt). - // 3. This node's AdvertiseIP (single-node default install) → AdvertiseIP. // 4. DevNetworking shim → 10.0.2.2. // 5. AWSGW bound to specific IP (no br-mgmt, no advertise) → that IP. // 6. Else: error and skip assignment — no silent empty URL. @@ -2802,17 +2812,20 @@ func (d *Daemon) wireLBAgentConfig() { advertiseIP := d.config.AdvertiseIP switch { - case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && !net.ParseIP(awsgwBindIP).IsLoopback(): - // Multi-node: AWSGW listens on a specific WAN IP. Use that IP as the - // gateway URL; internal LBs get a host route via br-mgmt. + case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && + !net.ParseIP(awsgwBindIP).IsLoopback() && awsgwBindIP != advertiseIP: + // Multi-node: AWSGW on a dedicated mgmt IP. VMs can't reach it via + // VPC → external, so add a bootcmd host route via br-mgmt. gatewayHost = awsgwBindIP d.mgmtRouteVia = awsgwBindIP - case d.mgmtBridgeIP != "": - // Multi-node with AWSGW on 0.0.0.0 — br-mgmt reaches all LBs. - gatewayHost = d.mgmtBridgeIP case advertiseIP != "" && advertiseIP != "0.0.0.0": - // Single-node default install — off-host clients dial the advertised IP. + // Single-node, or multi-node where AWSGW binds to AdvertiseIP: VMs + // reach AWSGW via the normal VPC → external path. No mgmt host route. gatewayHost = advertiseIP + case d.mgmtBridgeIP != "": + // br-mgmt present + AWSGW on 0.0.0.0 and no advertiseIP — br-mgmt IP + // is the only reachable address. + gatewayHost = d.mgmtBridgeIP case d.config.Daemon.DevNetworking: gatewayHost = "10.0.2.2" case awsgwBindIP != "" && awsgwBindIP != "0.0.0.0": diff --git a/spinifex/daemon/daemon_wire_lb_test.go b/spinifex/daemon/daemon_wire_lb_test.go index 21fdab452..60e589957 100644 --- a/spinifex/daemon/daemon_wire_lb_test.go +++ b/spinifex/daemon/daemon_wire_lb_test.go @@ -202,3 +202,44 @@ func TestWireLBAgentConfig_GatewayURL_NoAdvertiseNoBridge(t *testing.T) { assert.Empty(t, d.elbv2Service.GatewayURL) } + +// Single-node local dev: AWSGW on 0.0.0.0, br-mgmt present, AdvertiseIP set. +// Gateway must be AdvertiseIP (reachable via the VM's WAN NIC), NOT the +// br-mgmt IP (guest has no DHCP on br-mgmt so that subnet is unreachable). +func TestWireLBAgentConfig_GatewayURL_AdvertiseOverMgmtBridge(t *testing.T) { + cfg := &config.Config{ + AdvertiseIP: "192.168.1.32", + AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"}, + } + d := newWireLBTestDaemon(t, cfg) + d.mgmtBridgeIP = "10.14.8.1" + + d.wireLBAgentConfig() + + assert.Equal(t, "https://192.168.1.32:9999", d.elbv2Service.GatewayURL) + assert.Empty(t, d.mgmtRouteVia) + assert.Empty(t, d.elbv2Service.MgmtRouteGateway) + assert.Empty(t, d.elbv2Service.MgmtRouteTarget) +} + +// Single-node with br-mgmt AND a specific AWSGW bind that matches AdvertiseIP +// (e.g. `spx admin init --bind $PRIMARY_WAN`): the WAN IP is reachable via the +// normal VPC → external path, so the bootcmd mgmt host route must NOT be +// injected. Injecting it would divert the reply leg of host-initiated ALB +// connections out br-mgmt, bypassing OVN SNAT and arriving at the host with a +// source that doesn't match the open TCP socket. +func TestWireLBAgentConfig_GatewayURL_SingleNodeBindMatchesAdvertise(t *testing.T) { + cfg := &config.Config{ + AdvertiseIP: "192.168.0.17", + AWSGW: config.AWSGWConfig{Host: "192.168.0.17:9999"}, + } + d := newWireLBTestDaemon(t, cfg) + d.mgmtBridgeIP = "10.15.8.1" + + d.wireLBAgentConfig() + + assert.Equal(t, "https://192.168.0.17:9999", d.elbv2Service.GatewayURL) + assert.Empty(t, d.mgmtRouteVia) + assert.Empty(t, d.elbv2Service.MgmtRouteGateway) + assert.Empty(t, d.elbv2Service.MgmtRouteTarget) +} diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go index 3464680cd..e5dba061d 100644 --- a/spinifex/handlers/ec2/eip/service_impl.go +++ b/spinifex/handlers/ec2/eip/service_impl.go @@ -569,12 +569,17 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (* // publishNATEvent publishes a NAT lifecycle event to NATS for vpcd consumption. // This is fire-and-forget; errors are logged but do not fail the API response. +// +// PortName must match the OVN logical switch port name ("port-") because +// vpcd sets NAT.LogicalPort to this value in distributed NAT mode (direct +// bridge). A mismatch creates a dnat_and_snat row pointing at a nonexistent +// port, and OVN never programs the DNAT flow. func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, eniID, mac string) { utils.PublishEvent(s.natsConn, topic, natEvent{ VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, - PortName: eniID, + PortName: "port-" + eniID, MAC: mac, }) } diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go index 0a4689f65..5372d1a9e 100644 --- a/spinifex/handlers/ec2/eip/service_impl_test.go +++ b/spinifex/handlers/ec2/eip/service_impl_test.go @@ -3,6 +3,7 @@ package handlers_ec2_eip import ( "encoding/json" "testing" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -555,3 +556,32 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) { assert.Len(t, desc.Addresses, 1) assert.Equal(t, *out.AllocationId, *desc.Addresses[0].AllocationId) } + +// TestEIP_PublishNATEvent_PortNameHasPortPrefix is a regression test for a bug +// where EIPServiceImpl.publishNATEvent sent the raw ENI id as PortName. vpcd +// writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge), +// so a mismatch with the OVN logical switch port name ("port-") produces +// a dnat_and_snat row pointing at a nonexistent port — OVN never programs the +// flow and the EIP black-holes. See mulga-js-17. +func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) { + svc, _ := setupTestEIP(t) + + sub, err := svc.natsConn.SubscribeSync("vpc.add-nat") + require.NoError(t, err) + defer func() { _ = sub.Unsubscribe() }() + + const eniID = "eni-abc123" + svc.publishNATEvent("vpc.add-nat", "vpc-1", "198.51.100.10", "10.0.0.5", eniID, "02:00:00:00:00:01") + + msg, err := sub.NextMsg(2 * time.Second) + require.NoError(t, err) + + var got natEvent + require.NoError(t, json.Unmarshal(msg.Data, &got)) + assert.Equal(t, "port-"+eniID, got.PortName, + "PortName must match the OVN logical switch port name (port-); "+ + "otherwise vpcd's distributed NAT creates a dnat_and_snat rule "+ + "pointing at a nonexistent logical port and the flow is never programmed") + assert.Equal(t, "10.0.0.5", got.LogicalIP) + assert.Equal(t, "198.51.100.10", got.ExternalIP) +} diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index 715d259db..20e615d42 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -388,7 +388,7 @@ else fi # Wait for SSH to become ready (VM boot + cloud-init) -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Test basic SSH connectivity test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" @@ -1044,7 +1044,7 @@ fi echo "SSH endpoint: $SSH_HOST:$SSH_PORT" # Wait for SSH to become ready -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Verify vCPU count matches the new instance type (nproc reports online CPUs) echo "Verifying vCPU count inside the VM..." @@ -1119,7 +1119,7 @@ else SSH_PORT=$(get_ssh_port "$INSTANCE_ID") SSH_HOST=$(get_ssh_host "$INSTANCE_ID") fi -wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30 +wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60 # Verify guest actually rebooted (uptime < 120 seconds) UPTIME_SECS=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ diff --git a/tests/e2e/run-pseudo-multinode-e2e.sh b/tests/e2e/run-pseudo-multinode-e2e.sh index 1fad30130..ac40aebd2 100755 --- a/tests/e2e/run-pseudo-multinode-e2e.sh +++ b/tests/e2e/run-pseudo-multinode-e2e.sh @@ -385,7 +385,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do SSH_HOSTS+=("$SSH_HOST") # Wait for SSH to become ready (VM boot + cloud-init) - wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 30 + wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 60 # Test basic SSH connectivity test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh index a235f23db..a65c7bee2 100755 --- a/tests/e2e/run-tofu-examples-e2e.sh +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -5,6 +5,9 @@ # brought up a single-node cluster. For each workbook, the driver does a # clean tofu init/apply, runs one smoke assertion, then tofu destroy. # +# All assertions are minimal by design (one check per workbook) so the +# maintenance cost of keeping them in sync with workbook edits stays low. +# # Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the # tarball-included ./workbooks/ directory beside this script). @@ -16,12 +19,14 @@ export AWS_PROFILE=spinifex WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}" OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}" -# awsgw and predastore bind to the WAN IP (bootstrap-install passes --bind -# ${PRIMARY_WAN}), so workbooks need the WAN IP for both the tofu provider -# endpoints and any CLI assertions that talk to S3 (:8443). +# awsgw and predastore are bound to the WAN IP (bootstrap-install passes --bind +# ${PRIMARY_WAN}), not loopback. Workbooks need the WAN IP for both the tofu +# provider endpoints and any CLI assertions that talk to S3 (:8443). WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}') SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR) +CURRENT_WORKBOOK="" + log() { echo "[$(date +%H:%M:%S)] $*"; } install_tofu() { @@ -41,6 +46,37 @@ install_tofu() { rm -rf "$tmp" } +cleanup() { + EXIT_CODE=$? + if [ "$EXIT_CODE" -ne 0 ]; then + log "=== FAIL (workbook=${CURRENT_WORKBOOK:-}) — dumping diagnostics ===" + log "--- spx get vms ---" + sudo -u spinifex-daemon spx get vms 2>&1 | head -80 || \ + spx get vms 2>&1 | head -80 || true + log "--- aws ec2 describe-instances ---" + aws ec2 describe-instances \ + --query 'Reservations[].Instances[].[InstanceId,State.Name,StateReason.Message,PrivateIpAddress,PublicIpAddress,Tags[?Key==`Name`].Value|[0]]' \ + --output table 2>&1 | head -60 || true + log "--- spinifex service errors (last 15 min) ---" + for svc in spinifex-daemon spinifex-awsgw spinifex-vpcd spinifex-viperblock spinifex-predastore; do + sudo journalctl -u "$svc" --since '15 min ago' --no-pager 2>/dev/null | \ + grep -iE 'panic|fatal|level=error|ERROR' | tail -30 | \ + sed "s|^| [${svc}] |" || true + done + log "--- service journals (last 200 lines each) ---" + for svc in spinifex-nats spinifex-predastore spinifex-viperblock \ + spinifex-daemon spinifex-awsgw spinifex-vpcd; do + echo "=== ${svc} ===" + sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true + done + fi + exit "$EXIT_CODE" +} +trap cleanup EXIT + +# --- Per-workbook assertions --- + +# Wait up to 150s for instance SSH readiness. wait_for_ssh() { local key="$1" host="$2" for _ in $(seq 1 30); do @@ -52,11 +88,14 @@ wait_for_ssh() { return 1 } +# Wait up to $2 seconds (default 150) for a 200 response from $1. wait_for_http_200() { local url="$1" budget="${2:-150}" local attempts=$((budget / 5)) for _ in $(seq 1 "$attempts"); do - if [ "$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" = "200" ]; then + local status + status=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000") + if [ "$status" = "200" ]; then return 0 fi sleep 5 @@ -64,6 +103,7 @@ wait_for_http_200() { return 1 } +# Wait up to $2 seconds for the target group $1 to report at least one healthy target. wait_for_alb_healthy() { local tg_arn="$1" budget="${2:-300}" local attempts=$((budget / 10)) @@ -87,46 +127,76 @@ assert_bastion_private_subnet() { key="$(pwd)/bastion-demo.pem" chmod 600 "$key" - wait_for_ssh "$key" "$bastion" || return 1 - # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem, - # so wait for the hop key before attempting the inner ssh. - ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ - 'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1' || return 1 + wait_for_ssh "$key" "$bastion" || { + log " bastion SSH not ready" + return 1 + } + # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem. + # Wait up to 180s for user_data to drop the key before hopping. + if ! ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ + 'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then + log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)" + return 1 + fi ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ~/.ssh/bastion-demo.pem ec2-user@${private} id" \ | grep -q '^uid=' } -assert_nginx_webserver() { - wait_for_http_200 "http://$(tofu output -raw public_ip)/" -} - assert_nginx_alb() { # ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README # documents fetching the public IP via elbv2 describe-load-balancers. - local name ip tg_arn + local name ip tg_arn attempt name=$(tofu output -raw alb_name) - ip=$(aws elbv2 describe-load-balancers --names "$name" \ - --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \ - --output text 2>/dev/null | awk '{print $1}') + + # Retry describe-load-balancers: the gateway intermittently returns + # InternalError during ALB provisioning while the lb-agent VM is booting. + ip="" + for attempt in 1 2 3 4 5 6; do + ip=$(aws elbv2 describe-load-balancers --names "$name" \ + --query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \ + --output text 2>/dev/null | awk '{print $1}') + if [ -n "$ip" ] && [ "$ip" != "None" ]; then + break + fi + log " nginx-alb: describe-load-balancers attempt ${attempt} returned no IP, retrying in 10s" + sleep 10 + done if [ -z "$ip" ] || [ "$ip" = "None" ]; then - log " nginx-alb: no public IP for ${name}" + log " nginx-alb: no public IP for ${name} after retries" return 1 fi + log " nginx-alb: ALB ${name} public IP ${ip}" + tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \ --query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null) if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then log " nginx-alb: could not resolve target group ARN" return 1 fi - wait_for_alb_healthy "$tg_arn" 300 || return 1 - wait_for_http_200 "http://${ip}/" 300 + + wait_for_alb_healthy "$tg_arn" 300 || { + log " nginx-alb: no healthy targets after 300s" + return 1 + } + + wait_for_http_200 "http://${ip}/" 300 || { + log " nginx-alb: targets healthy but no HTTP 200 after 300s" + return 1 + } +} + +assert_nginx_webserver() { + local ip + ip=$(tofu output -raw public_ip) + wait_for_http_200 "http://${ip}/" } assert_s3_webapp() { - # aws s3 goes to predastore on :8443, not awsgw on :9999 — the workbook's - # provider sets s3 = predastore_endpoint but the CLI doesn't inherit that. - local bucket endpoint sentinel + # aws s3 goes to predastore on :8443, not the awsgw on :9999 the default + # profile points at — the workbook's provider sets s3 = predastore_endpoint + # but the CLI doesn't inherit that. + local bucket sentinel endpoint bucket=$(tofu output -raw bucket_name) endpoint="https://${WAN_IP}:8443" sentinel="spinifex-nightly-$(date +%s).txt" @@ -135,8 +205,9 @@ assert_s3_webapp() { } # Pick an instance type available on this cluster. Workbooks default to -# t3.small (Intel); on AMD-only hosts t3 isn't registered, so query and -# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM. +# t3.small (Intel); on AMD-only hosts t3 isn't registered, so we query and +# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM (matches the +# nginx-alb README's documented approach). detect_instance_type() { local endpoint="https://${WAN_IP}:9999" local picked @@ -152,6 +223,8 @@ detect_instance_type() { INSTANCE_TYPE="" +# --- Workbook driver --- + run_workbook() { local example="$1" local path="${WORKBOOK_DIR}/${example}" @@ -161,6 +234,7 @@ run_workbook() { return 1 fi + CURRENT_WORKBOOK="$example" log "=== ${example} ===" cd "$path" rm -rf .terraform terraform.tfstate* .terraform.lock.hcl @@ -171,7 +245,8 @@ run_workbook() { "-var=instance_type=${INSTANCE_TYPE}" ) - # s3-webapp has extra required vars; creds come from AWS_PROFILE=spinifex. + # s3-webapp has three required-no-default vars; creds come from + # AWS_PROFILE=spinifex so the workbook's boto3 client authenticates. if [ "$example" = "s3-webapp" ]; then local access_key secret_key access_key=$(aws configure get aws_access_key_id --profile spinifex) @@ -210,6 +285,8 @@ run_workbook() { return "$rc" } +# --- Main --- + install_tofu || { log "tofu install failed"; exit 1; } INSTANCE_TYPE=$(detect_instance_type) @@ -219,16 +296,12 @@ if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then fi log "Using instance_type=${INSTANCE_TYPE}" -# nginx-alb is disabled pending: (1) vpcd port-group live OVN client -# (CreatePortGroup/AddACL not yet implemented → SG rules are no-ops), -# (2) viperblock zero-write block-alignment bug that crashes lb-agent boot, -# (3) external-IP allocator reusing an ENI EIP for the ALB. Re-enable once -# those land. -for workbook in bastion-private-subnet nginx-webserver s3-webapp; do +for workbook in nginx-alb bastion-private-subnet nginx-webserver s3-webapp; do if ! run_workbook "$workbook"; then log "FAIL ${workbook} — aborting remaining workbooks" exit 1 fi done +CURRENT_WORKBOOK="" log "All workbooks passed"