diff --git a/.github/scripts/internal-api-test-seconds.txt b/.github/scripts/internal-api-test-seconds.txt index d3465b3b52..049f78cbac 100644 --- a/.github/scripts/internal-api-test-seconds.txt +++ b/.github/scripts/internal-api-test-seconds.txt @@ -1,212 +1,217 @@ -# Estimated -race run time in seconds of the slowest internal/api top-level -# tests on a GitHub ubuntu-24.04 runner. Used only by -# select-internal-api-shard.sh to choose where the Backend tests (api-N) -# shards cut go test's list. Tests not named here weigh DEFAULT_WEIGHT (0.05s, -# the measured mean of the unlisted tests). A missing, stale or renamed entry -# can make the shards uneven but can never drop or repeat a test. +# Run time in seconds of the slowest internal/api top-level tests under -race +# on a GitHub ubuntu-24.04 runner. Used only by select-internal-api-shard.sh +# to choose where the Backend tests (api-N) shards cut go test's list. Tests +# not named here weigh DEFAULT_WEIGHT, the mean of the tests under 0.5s. A +# missing, stale or renamed entry can make the shards uneven but can never +# drop or repeat a test. # -# Refresh when one api shard runs well over the others: run -# GOMAXPROCS=4 go test -race -json ./internal/api -# and list every top-level test at 0.5s or more. Seeded 2026-09-29 from such a -# local run scaled to the four shard totals of Build and Test run 36560070496 -# (the contract mock chart test runs about 1.4x slower on the runner, the -# wait-bound integration server tail about 0.8x). -TestActionRunnerRotationProductionRouterTLSPersistenceRestart 0.69 -TestMockActionInboxAndDetailUseCanonicalResponses 2.40 -TestMockActionInboxHonorsLimitAndLegacyPendingProjection 1.79 -TestMockActionMutationsFailReadOnly 0.98 -TestAgentReportHandlersClassifyBodyFailures 1.10 -TestStart_MockModeStartsChatServiceWhenPersistedConfigDisabled 0.95 -TestRestart_MockModeStartsChatServiceWhenPersistedConfigDisabled 0.95 -TestHandleChat_EmitsIdleProgressWhileServiceIsSilent 2.40 -TestAISettingsHandler_ProviderSettersConcurrentServiceCreation 8.31 -TestHandleSaveGuestNote_VerifySavedFields 0.52 -TestHandleClearGuestKnowledge_OversizedBody 0.57 -TestHandlePatrolStream_LastEventIDHeaderTakesPrecedenceOverQueryFallback 1.99 -TestHandleExecuteStream_LicenseRequired 1.05 -TestHandleExecuteStream_PromptRequired 1.06 -TestHandleExecuteStream_Success 1.80 -TestHandleExecuteStream_EmitsIdleProgressWhileServiceIsSilent 1.74 -TestHandleInvestigateAlert_EmitsIdleProgressWhileServiceIsSilent 1.69 -TestAISettingsHandler_PatrolReadinessBranches 0.85 -TestHandleInvestigateAlert_UsesVisibleIdleProgressStream 1.73 -TestHandleGetPatterns_UnlockedWithData 1.04 -TestHandleGetPredictions_WithData 1.04 -TestHandleGetCorrelations_WithData 1.05 -TestHandleGetCorrelations_UsesCanonicalIntelligenceFacade 1.04 -TestExtractAndStoreAuthContext_UsesTenantConfigForToken 2.98 -TestAuthDenialPruneKeepsTrackedSetBounded 1.92 -TestAvailabilityHandlersListReturnsMockTargetsInMockMode 1.03 -TestAvailabilityHandlersTestSavedMockTargetUsesSyntheticStatus 1.03 -TestChangePasswordRefusesNonAdminSession 0.83 -TestChangePasswordAllowsConfiguredAdminSession 1.52 -TestChangePasswordBasicAuthPathUnaffected 4.19 -TestChangePasswordFixtureOwnsRouterLifecycle 0.84 -TestConfigTransferAnonymousAuthenticatedModesDenyBeforeBodyRead 0.50 -TestConfigTransferAuthorizedInstanceModesReachHandler 0.70 -TestConnectionsHandleListIncludesMockAvailabilityTargets 1.13 -TestConnectionsHandleListDropsRealSourcesInMockMode 1.19 -TestConnectionsLedgerKeepsRealSourcesWhenRealPollingRetained 1.15 -TestContract_MockAvailabilityTargetsUseSavedTargetPayloads 1.13 -TestContract_MockDiscoveryEndpointsUseCanonicalPayloads 1.05 -TestContract_MockChartRoutesUseCanonicalMockUnifiedReadStateForVMwareHosts 391.48 -TestContract_PlatformMockToggleRebindsRuntimeConnectionsAndResources 19.96 -TestContract_PlatformMockConnectionListsUseSharedFixtureMetadata 1.08 -TestContract_RecoveryPointsMockPathReturnsCanonicalProviderBackedFixtures 1.61 -TestContract_SecurityStatusPresentationPolicyProEditionCommercialContext 0.50 -TestContract_StateAndResourceListShareCanonicalMockResourceContract 4.64 -TestContract_AgentCommandConfigGateMirrorsChannelAdmission 1.13 -TestContract_HostedDiagnosticsDockerPrepareTokenValidatesOriginBeforeMutation 1.22 -TestContract_HostedInstallerOriginsFailClosedAtRouter 4.04 -TestContract_RuntimeDisplayServesPresentationValuesWithoutAdmin 0.84 -TestSecurityStatusCurrentUserForScopedLocalSession 0.84 -TestSecurityStatusOrgBoundSettingsFollowOrgManagementRule 0.84 -TestSecurityStatusOrgBoundReportingRespectsAuthorizer 0.83 -TestDiscoveryHandlers_MockModeServesFixturesWithoutService 1.13 -TestDockerAgentReportSizeContractAcceptsOrdinaryAndLargeFleets 0.55 -TestDockerAgentReportSizeContractBoundaries 2.57 -TestCollectorUninstallTransactionFailsClosedAndRetriesAfterRestart 2.15 -TestHostedTenantAgentInstallCommand_GeneratesOrgBoundTokenAndCommand 3.13 -TestGenerateHostedTenantAgentInstallCommandEnforcesHostedOrgBoundary 1.11 -TestHostedTenantAgentInstallCommandCarriesCommandPolicyIntent 1.49 -TestHostedTenantAgentInstallTokenCannotReportToOtherTenant 0.50 -TestIssue1753SameNameProxmoxAgentsAuthenticateAndStayDistinctEndToEnd 2.20 -TestSyncReleaseDemoFixtureRuntime_DoesNotPoliceMockFixturesInDevBuilds 1.23 -TestMetricsHistoryFallbackSynthesizesUnseededMockWorkloadHistory 1.64 -TestMetricsHistoryFallbackSynthesizesMockHostAndNodeDrawerSeries 1.99 -TestMetricsHistoryFallbackSupplementsPartialMockStoreSeries 1.68 -TestMetricsHistoryFallbackMockDiskSynthesizesSeries 1.65 -TestMetricsHistoryFallbackMockUnifiedStorageSynthesizesSeries 2.27 -TestMetricsHistoryFallbackMockStorageBypassesSparseStoreSeries 2.47 -TestSSOOnlyExplicitAdminReachesEveryAdminGuard 1.42 -TestUnrelatedSSOUserStillRefusedWhenLocalAdminConfigured 2.10 -TestPatrolActionBrokerAutonomyModeIsAnUpperBound 0.53 -TestPlatformAdminRouteRefusesUnrelatedSSOUser 0.84 -TestJourneyGeneralAPIBudgetIsolation 1.75 -TestTenantRBACProvider_ConcurrentGetAndRemove 0.52 -TestTenantRBACProvider_NoConnectionLeak 1.43 -TestHandleListPointsAcceptsCanonicalPlatformQuery 1.82 -TestHandleListRollupsExposeCanonicalPlatformsPayload 1.86 -TestHandleListRollupsMetaReportsNormalizedPagination 2.71 -TestHandleListPointsMetaReportsNormalizedPagination 2.74 -TestStartPatrolForContext_DoesNotOverwriteOtherTenantPatrolComponents 2.30 -TestStartPatrolForContext_RejectsMismatchedAIServiceOrg 1.24 -TestRouterMockMode_SeedsTrueNASAndVMwareSupplementalResources 1.58 -TestRouterMockMode_SeedsVMwareSupplementalActivity 1.21 -TestRouteExecuteStream_Success 0.94 -TestRouteExecuteStream_MethodNotAllowed 0.76 -TestRouteExecuteStream_NoAuth 0.60 -TestRouteExecuteStream_WrongScope 0.65 -TestRouteExecuteStream_EmptyPrompt 0.83 -TestRouteExecuteStream_WhitespaceOnlyPrompt 0.73 -TestRouteExecuteStream_InvalidJSON 0.80 -TestRouteExecuteStream_AIDisabled 0.78 -TestRouteExecuteStream_AutofixLicenseRequired 0.75 -TestRouteExecuteStream_RemediationLicenseRequired 0.66 -TestRouteExecuteStream_OversizedBody 0.82 -TestRouteExecuteStream_UseCaseCaseInsensitive 0.86 -TestRouteExecuteStream_SSEHeaders 0.66 -TestRouteExecuteStream_CompleteEventBeforeDone 0.76 -TestRouteExecuteStream_EmptyBody 0.68 -TestRouteExecuteStream_NullPrompt 0.74 -TestRouteExecuteStream_EmptyJSONObject 0.55 -TestRouteExecuteStream_ServiceError 0.69 -TestRouteExecuteStream_InvalidTargetType 0.60 -TestRouteExecuteStream_InvalidHistoryRole 0.70 -TestRouteExecuteStream_TargetIDTooLong 0.58 -TestRouteExecuteStream_ValidTargetTypes 2.08 -TestRouteExecuteStream_EmptyTargetTypeAllowed 1.50 -TestRouteExecuteStream_WithConversationHistory 0.58 -TestRouteExecuteStream_WithTargetContext 0.75 -TestRouteExecuteStream_DefaultUseCaseIsChat 0.73 -TestRouteExecuteStream_FindingIDPassedThrough 0.73 -TestRouteExecuteStream_ModelFieldPassedThrough 0.63 -TestRouteExecute_Success 0.74 -TestRouteExecute_MethodNotAllowed 0.75 -TestRouteExecute_NoAuth 0.79 -TestRouteExecute_WrongScope 0.69 -TestRouteExecute_EmptyPrompt 0.74 -TestRouteExecute_InvalidJSON 0.75 -TestRouteExecute_WithConversationHistory 0.68 -TestRouteExecute_WithTargetContext 0.87 -TestRouteExecute_DefaultUseCaseIsChat 0.73 -TestRouteExecute_AutofixLicenseRequired 0.75 -TestRouteExecute_RemediationLicenseRequired 0.56 -TestRouteExecute_OversizedBody 0.93 -TestRouteExecute_EmptyBody 0.66 -TestRouteExecute_EmptyJSONObject 0.94 -TestRouteExecute_NullPrompt 0.75 -TestRouteExecute_ServiceError 0.73 -TestRouteExecute_InvalidTargetType 0.75 -TestRouteExecute_ValidTargetTypes 2.17 -TestRouteExecute_EmptyTargetTypeAllowed 0.75 -TestRouteExecute_InvalidHistoryRole 0.73 -TestRouteExecute_FindingIDPassedThrough 0.76 -TestRouteExecute_ModelFieldPassedThrough 0.73 -TestRouteListModels_Success 0.68 -TestRouteListModels_MethodNotAllowed 0.66 -TestRouteListModels_WrongScope 0.72 -TestRouteListModels_ProviderError 0.75 -TestRouteListModels_EmptyModelList 0.76 -TestRouteListModels_ResponseContentType 0.63 -TestRouteListModels_CachedField 0.76 -TestRouteListModels_MalformedProviderResponse 0.78 -TestRouteListModels_HEADMethod 0.87 -TestRouteTestConnection_Success 0.76 -TestRouteTestConnection_MethodNotAllowed 0.72 -TestRouteTestConnection_NoAuth 0.74 -TestRouteTestConnection_WrongScope 0.74 -TestRouteTestProvider_Success 0.72 -TestRouteTestProvider_UnconfiguredProvider 0.66 -TestRouteTestProvider_ConnectionFailure 0.60 -TestRouteTestProvider_MethodNotAllowed 0.76 -TestRouteTestProvider_NoAuth 0.75 -TestRouteTestProvider_MultipleProviders 0.74 -TestRouteTestProvider_UnknownProvider 0.60 -TestRouteTestProvider_WrongScope 0.75 -TestRouteTestProvider_InvalidProviderNames 0.74 -TestRouteTestProvider_PathTraversal 0.85 -TestRouter_HandleState_MockIsolation 4.14 -TestAITestConnectionRouteWithValidScope 0.75 -TestAITestConnectionRouteRejectsWrongScope 0.82 -TestAITestProviderRouteRejectsWrongScope 0.74 -TestPublicEndpointsBypassAuthInAPIMode 0.61 -TestSettingsCapabilitiesMatchRouteEnforcementWithoutRBAC 0.86 -TestSettingsCapabilitiesGrantConfiguredAdminWithoutRBAC 0.93 -TestTrueNASHandlers_HandleList_ReturnsMockConnectionsInMockMode 1.23 -TestFreeBSDOfflineAgentPreflight 1.83 -TestBranchcov0723AmExtractFromZip 8.86 -TestHandleApplyUpdate_BlocksWhenReadinessBlocked 1.47 -TestHandleGetUpdatePlan 1.65 -TestHandleGetUpdatePlan_IncludesUpgradeReadiness 1.34 -TestHandleGetUpdatePlan_InvalidChannel 1.09 -TestHandleGetUpdatePlan_PrepareError 1.43 -TestVMwareHandlers_HandleList_ReturnsMockConnectionsInMockMode 1.33 -TestAlertsEndpoints 20.56 -TestRecoveryRollupsEndpointToleratesMalformedPersistedMetadata 0.55 -TestHealthEndpoint 19.04 -TestVersionEndpointUsesRepoVersion 19.47 -TestAlertAcknowledge_AllowsPrintableAlertIDs 18.81 -TestStateEndpointReturnsMockData 20.72 -TestRecoveryRollupsEndpointReturnsMockData 20.65 -TestProtectedEndpointsRequireAuthentication 18.62 -TestAPIOnlyModeRequiresToken 21.79 -TestServerInfoEndpointReportsDevelopment 18.85 -TestRecoveryPointsEndpointReturnsMockData 21.75 -TestServerInfoEndpointMethodNotAllowed 18.65 -TestHealthEndpointMethodNotAllowed 18.76 -TestHealthEndpointHeadAllowed 18.68 -TestConfigNodesUsesMockTopology 17.84 -TestMockModeToggleEndpoint 25.43 -TestMockModeToggleFromRealDataRebindsPlatformSupplementalProviders 19.24 -TestAuthenticatedEndpointsRequireToken 19.40 -TestAPITokenQueryAndHeaderAuth 23.11 -TestRecoveryEndpointRequiresDirectLoopback 20.32 -TestWebSocketSendsInitialState 19.38 -TestWebsocketPayloadContractShape 17.62 -TestWebsocketPayloadUsesCanonicalStateContract 19.56 -TestSessionCookieAllowsAuthenticatedAccess 20.16 -TestRevokedAPITokenImmediatelyLosesAccess 24.33 -TestLimitedAPITokenCannotCreateBroaderToken 20.31 -TestPublicURLDetectionUsesForwardedHeaders 17.31 -TestPublicURLDetectionRespectsEnvOverride 19.14 +# Every api shard uploads an internal-api-test-seconds-N artifact with the +# seconds of each test it ran on the runner, so refresh this file from recent +# passing Build and Test runs with +# python3 .github/scripts/refresh-internal-api-test-seconds.py --run [--run ] +# when one api shard runs well over the others. +# +# Seeded 2026-09-29, before any such artifact existed, from a local +# GOMAXPROCS=4 go test -race -json run with each region of go test's order +# scaled so its sum matches the seconds the runner reported for it. +# Equal-count quarters (runs 36557693864, 36560070496, 36567708327, +# 36577491881) averaged 107s, 437s, 100s and 787s. The weighted cut (runs +# 36566872567, 36578019732) averaged 131s for the first 1187 tests, 350s for +# the next 23 and 608s for the last 21. Its 2556-test shard (887s, 875s) is +# left out, because it ran through -run . -skip, which recompiles the skip +# pattern for every test and subtest. The last quarter and the last 21 tests +# disagree on how the tail splits, so the two tail regions take the midpoint +# of an exact fit and one pooled factor. +DEFAULT_WEIGHT 0.05 +TestActionRunnerRotationProductionRouterTLSPersistenceRestart 0.82 +TestMockActionInboxAndDetailUseCanonicalResponses 2.85 +TestMockActionInboxHonorsLimitAndLegacyPendingProjection 2.12 +TestMockActionMutationsFailReadOnly 1.16 +TestAgentReportHandlersClassifyBodyFailures 1.31 +TestStart_MockModeStartsChatServiceWhenPersistedConfigDisabled 1.13 +TestRestart_MockModeStartsChatServiceWhenPersistedConfigDisabled 1.13 +TestHandleChat_EmitsIdleProgressWhileServiceIsSilent 2.85 +TestAISettingsHandler_ProviderSettersConcurrentServiceCreation 9.86 +TestHandleSaveGuestNote_VerifySavedFields 0.62 +TestHandleClearGuestKnowledge_OversizedBody 0.68 +TestHandlePatrolStream_LastEventIDHeaderTakesPrecedenceOverQueryFallback 2.37 +TestHandleExecuteStream_LicenseRequired 1.25 +TestHandleExecuteStream_PromptRequired 1.26 +TestHandleExecuteStream_Success 2.13 +TestHandleExecuteStream_EmitsIdleProgressWhileServiceIsSilent 2.06 +TestHandleInvestigateAlert_EmitsIdleProgressWhileServiceIsSilent 2.01 +TestAISettingsHandler_PatrolReadinessBranches 1.01 +TestAISettingsHandler_KeylessOpenAICompatibleSetupIsConfiguredAndValidated 0.51 +TestAISettingsHandler_TestProvider_ConnectionFailure 0.52 +TestHandleGetAICostSummary_MethodNotAllowed 0.54 +TestHandleInvestigateAlert_UsesVisibleIdleProgressStream 2.05 +TestHandleGetPatterns_UnlockedWithData 1.23 +TestHandleGetPredictions_WithData 1.23 +TestHandleGetCorrelations_WithData 1.25 +TestHandleGetCorrelations_UsesCanonicalIntelligenceFacade 1.24 +TestExtractAndStoreAuthContext_UsesTenantConfigForToken 3.54 +TestAuthDenialPruneKeepsTrackedSetBounded 2.28 +TestAvailabilityHandlersListReturnsMockTargetsInMockMode 1.22 +TestAvailabilityHandlersTestSavedMockTargetUsesSyntheticStatus 1.22 +TestChangePasswordRefusesNonAdminSession 0.90 +TestChangePasswordAllowsConfiguredAdminSession 1.66 +TestChangePasswordBasicAuthPathUnaffected 4.57 +TestChangePasswordFixtureOwnsRouterLifecycle 0.91 +TestConfigTransferAnonymousAuthenticatedModesDenyBeforeBodyRead 0.55 +TestConfigTransferAuthorizedInstanceModesReachHandler 0.77 +TestConfigTransferTenantSessionsRequireManagement 0.54 +TestConnectionsHandleListIncludesMockAvailabilityTargets 1.23 +TestConnectionsHandleListDropsRealSourcesInMockMode 1.30 +TestConnectionsLedgerKeepsRealSourcesWhenRealPollingRetained 1.26 +TestContract_MockAvailabilityTargetsUseSavedTargetPayloads 1.23 +TestContract_MockDiscoveryEndpointsUseCanonicalPayloads 1.15 +TestContract_MockChartRoutesUseCanonicalMockUnifiedReadStateForVMwareHosts 324.29 +TestContract_PlatformMockToggleRebindsRuntimeConnectionsAndResources 24.19 +TestContract_PlatformMockConnectionListsUseSharedFixtureMetadata 1.31 +TestContract_RecoveryPointsMockPathReturnsCanonicalProviderBackedFixtures 2.15 +TestContract_SecurityStatusPresentationPolicyProEditionCommercialContext 0.67 +TestContract_StateAndResourceListShareCanonicalMockResourceContract 6.16 +TestContract_AgentCommandConfigGateMirrorsChannelAdmission 1.50 +TestContract_HostedDiagnosticsDockerPrepareTokenValidatesOriginBeforeMutation 1.62 +TestContract_HostedInstallerOriginsFailClosedAtRouter 5.36 +TestContract_RuntimeDisplayServesPresentationValuesWithoutAdmin 1.11 +TestSecurityStatusCurrentUserForScopedLocalSession 1.11 +TestSecurityStatusOrgBoundSettingsFollowOrgManagementRule 1.11 +TestSecurityStatusOrgBoundReportingRespectsAuthorizer 1.10 +TestDiscoveryHandlers_MockModeServesFixturesWithoutService 1.50 +TestDockerAgentReportSizeContractAcceptsOrdinaryAndLargeFleets 0.73 +TestDockerAgentReportSizeContractBoundaries 3.42 +TestHostAgentRemovalLifecycleThroughAuthenticatedRouterAndRestart 0.59 +TestCollectorUninstallTransactionFailsClosedAndRetriesAfterRestart 2.85 +TestHostedTenantAgentInstallCommand_GeneratesOrgBoundTokenAndCommand 4.15 +TestGenerateHostedTenantAgentInstallCommandEnforcesHostedOrgBoundary 1.48 +TestHostedTenantAgentInstallCommandCarriesCommandPolicyIntent 1.98 +TestHostedTenantAgentInstallTokenCannotReportToOtherTenant 0.67 +TestIssue1753SameNameProxmoxAgentsAuthenticateAndStayDistinctEndToEnd 1.73 +TestSyncReleaseDemoFixtureRuntime_DoesNotPoliceMockFixturesInDevBuilds 0.96 +TestMetricsHistoryFallbackSynthesizesUnseededMockWorkloadHistory 1.29 +TestMetricsHistoryFallbackSynthesizesMockHostAndNodeDrawerSeries 1.57 +TestMetricsHistoryFallbackSupplementsPartialMockStoreSeries 1.32 +TestMetricsHistoryFallbackMockDiskSynthesizesSeries 1.30 +TestMetricsHistoryFallbackMockUnifiedStorageSynthesizesSeries 1.79 +TestMetricsHistoryFallbackMockStorageBypassesSparseStoreSeries 1.94 +TestSSOOnlyExplicitAdminReachesEveryAdminGuard 1.11 +TestUnrelatedSSOUserStillRefusedWhenLocalAdminConfigured 1.65 +TestPlatformAdminRouteRefusesUnrelatedSSOUser 0.66 +TestJourneyGeneralAPIBudgetIsolation 1.38 +TestTenantRBACProvider_NoConnectionLeak 1.13 +TestHandleListPointsAcceptsCanonicalPlatformQuery 1.43 +TestHandleListRollupsExposeCanonicalPlatformsPayload 1.46 +TestHandleListRollupsMetaReportsNormalizedPagination 2.13 +TestHandleListPointsMetaReportsNormalizedPagination 2.15 +TestStartPatrolForContext_DoesNotOverwriteOtherTenantPatrolComponents 1.81 +TestStartPatrolForContext_RejectsMismatchedAIServiceOrg 0.98 +TestRouterMockMode_SeedsTrueNASAndVMwareSupplementalResources 1.24 +TestRouterMockMode_SeedsVMwareSupplementalActivity 0.95 +TestRouteExecuteStream_Success 0.74 +TestRouteExecuteStream_MethodNotAllowed 0.60 +TestRouteExecuteStream_WrongScope 0.51 +TestRouteExecuteStream_EmptyPrompt 0.65 +TestRouteExecuteStream_WhitespaceOnlyPrompt 0.58 +TestRouteExecuteStream_InvalidJSON 0.63 +TestRouteExecuteStream_AIDisabled 0.61 +TestRouteExecuteStream_AutofixLicenseRequired 0.59 +TestRouteExecuteStream_RemediationLicenseRequired 0.52 +TestRouteExecuteStream_OversizedBody 0.64 +TestRouteExecuteStream_UseCaseCaseInsensitive 0.68 +TestRouteExecuteStream_SSEHeaders 0.52 +TestRouteExecuteStream_CompleteEventBeforeDone 0.60 +TestRouteExecuteStream_EmptyBody 0.54 +TestRouteExecuteStream_NullPrompt 0.58 +TestRouteExecuteStream_ServiceError 0.55 +TestRouteExecuteStream_InvalidHistoryRole 0.55 +TestRouteExecuteStream_ValidTargetTypes 1.64 +TestRouteExecuteStream_EmptyTargetTypeAllowed 1.18 +TestRouteExecuteStream_WithTargetContext 0.59 +TestRouteExecuteStream_DefaultUseCaseIsChat 0.58 +TestRouteExecuteStream_FindingIDPassedThrough 0.58 +TestRouteExecute_Success 0.58 +TestRouteExecute_MethodNotAllowed 0.59 +TestRouteExecute_NoAuth 0.62 +TestRouteExecute_WrongScope 0.55 +TestRouteExecute_EmptyPrompt 0.58 +TestRouteExecute_InvalidJSON 0.59 +TestRouteExecute_WithConversationHistory 0.54 +TestRouteExecute_WithTargetContext 0.69 +TestRouteExecute_DefaultUseCaseIsChat 0.58 +TestRouteExecute_AutofixLicenseRequired 0.59 +TestRouteExecute_OversizedBody 0.73 +TestRouteExecute_EmptyBody 0.52 +TestRouteExecute_EmptyJSONObject 0.74 +TestRouteExecute_NullPrompt 0.59 +TestRouteExecute_ServiceError 0.58 +TestRouteExecute_InvalidTargetType 0.59 +TestRouteExecute_ValidTargetTypes 1.70 +TestRouteExecute_EmptyTargetTypeAllowed 0.59 +TestRouteExecute_InvalidHistoryRole 0.58 +TestRouteExecute_FindingIDPassedThrough 0.60 +TestRouteExecute_ModelFieldPassedThrough 0.58 +TestRouteListModels_Success 0.54 +TestRouteListModels_MethodNotAllowed 0.52 +TestRouteListModels_WrongScope 0.57 +TestRouteListModels_ProviderError 0.59 +TestRouteListModels_EmptyModelList 0.60 +TestRouteListModels_CachedField 0.60 +TestRouteListModels_MalformedProviderResponse 0.61 +TestRouteListModels_HEADMethod 0.69 +TestRouteTestConnection_Success 0.60 +TestRouteTestConnection_MethodNotAllowed 0.57 +TestRouteTestConnection_NoAuth 0.58 +TestRouteTestConnection_WrongScope 0.58 +TestRouteTestProvider_Success 0.57 +TestRouteTestProvider_UnconfiguredProvider 0.52 +TestRouteTestProvider_MethodNotAllowed 0.60 +TestRouteTestProvider_NoAuth 0.59 +TestRouteTestProvider_MultipleProviders 0.58 +TestRouteTestProvider_WrongScope 0.59 +TestRouteTestProvider_InvalidProviderNames 0.58 +TestRouteTestProvider_PathTraversal 0.67 +TestRouter_HandleState_MockIsolation 3.26 +TestAITestConnectionRouteWithValidScope 0.72 +TestAITestConnectionRouteRejectsWrongScope 0.79 +TestAITestProviderRouteRejectsWrongScope 0.71 +TestPublicEndpointsBypassAuthInAPIMode 0.59 +TestSettingsCapabilitiesMatchRouteEnforcementWithoutRBAC 0.83 +TestSettingsCapabilitiesGrantConfiguredAdminWithoutRBAC 0.90 +TestTrueNASHandlers_HandleList_ReturnsMockConnectionsInMockMode 1.18 +TestFreeBSDOfflineAgentPreflight 1.77 +TestBranchcov0723AmExtractFromZip 8.53 +TestHandleApplyUpdate_BlocksWhenReadinessBlocked 1.42 +TestHandleGetUpdatePlan 1.59 +TestHandleGetUpdatePlan_IncludesUpgradeReadiness 1.29 +TestHandleGetUpdatePlan_InvalidChannel 1.05 +TestHandleGetUpdatePlan_PrepareError 1.38 +TestVMwareHandlers_HandleList_ReturnsMockConnectionsInMockMode 1.28 +TestAlertsEndpoints 22.93 +TestRecoveryRollupsEndpointToleratesMalformedPersistedMetadata 0.53 +TestHealthEndpoint 21.24 +TestVersionEndpointUsesRepoVersion 21.71 +TestAlertAcknowledge_AllowsPrintableAlertIDs 20.98 +TestStateEndpointReturnsMockData 23.11 +TestRecoveryRollupsEndpointReturnsMockData 23.03 +TestProtectedEndpointsRequireAuthentication 20.77 +TestAPIOnlyModeRequiresToken 31.38 +TestServerInfoEndpointReportsDevelopment 27.15 +TestRecoveryPointsEndpointReturnsMockData 31.33 +TestServerInfoEndpointMethodNotAllowed 26.85 +TestHealthEndpointMethodNotAllowed 27.02 +TestHealthEndpointHeadAllowed 26.90 +TestConfigNodesUsesMockTopology 25.70 +TestMockModeToggleEndpoint 36.62 +TestMockModeToggleFromRealDataRebindsPlatformSupplementalProviders 27.70 +TestAuthenticatedEndpointsRequireToken 27.94 +TestAPITokenQueryAndHeaderAuth 33.28 +TestRecoveryEndpointRequiresDirectLoopback 29.26 +TestWebSocketSendsInitialState 27.92 +TestWebsocketPayloadContractShape 25.38 +TestWebsocketPayloadUsesCanonicalStateContract 28.16 +TestSessionCookieAllowsAuthenticatedAccess 29.04 +TestRevokedAPITokenImmediatelyLosesAccess 35.04 +TestLimitedAPITokenCannotCreateBroaderToken 29.25 +TestPublicURLDetectionUsesForwardedHeaders 24.93 +TestPublicURLDetectionRespectsEnvOverride 27.56 diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.py new file mode 100755 index 0000000000..8006abb945 --- /dev/null +++ b/.github/scripts/record-internal-api-test-seconds.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Print `go test -json` like plain `go test` and record per-test seconds. + +Usage: go test -json ... | record-internal-api-test-seconds.py + +`go test -json` reports every test the way `-v` does, which for internal/api +means thousands of lines per shard. This filter keeps the log as readable as +plain `go test`: package lines (build errors, `ok`, `FAIL`, panics outside a +test) print straight away, and a test's own output is held until the test +ends, printed if it failed and dropped if it passed or was skipped. Output of +a test that never finished (a panic or timeout killed the binary) is printed +at the end, so a crash is never hidden. + +Every finished top-level test is written to as +` `, after a `# package-seconds ` comment +holding the whole test binary's run time, the same shape as +.github/scripts/internal-api-test-seconds.txt, so +refresh-internal-api-test-seconds.py can rebuild the shard weights from CI. + +The exit status is non-zero when any test or package failed, but the caller +must still run with `set -o pipefail` so a `go test` failure that produced no +JSON (for example a vet or build error) fails the step. +""" + +from __future__ import annotations + +import json +import sys +from collections import defaultdict + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + + held: dict[str, list[str]] = defaultdict(list) + running: set[str] = set() + seconds: dict[str, float] = {} + package_seconds = None + failed = False + out = sys.stdout + + for raw in sys.stdin: + try: + event = json.loads(raw) + except ValueError: + event = None + if not isinstance(event, dict): + out.write(raw) + out.flush() + continue + + action = event.get("Action") + test = event.get("Test") or "" + top = test.split("/", 1)[0] + text = event.get("Output") or "" + + if not top: + # Package level: build output, `ok`/`FAIL` summary, panics and + # timeouts reported outside a test. + if text: + out.write(text) + out.flush() + if action == "fail": + failed = True + if action in ("pass", "fail") and isinstance(event.get("Elapsed"), (int, float)): + package_seconds = float(event["Elapsed"]) + continue + + if action == "run" and test == top: + running.add(top) + if text: + held[top].append(text) + if test != top or action not in ("pass", "fail", "skip"): + continue + + running.discard(top) + lines = held.pop(top, []) + if action == "fail": + failed = True + out.write("".join(lines)) + out.flush() + elapsed = event.get("Elapsed") + if isinstance(elapsed, (int, float)): + seconds[top] = float(elapsed) + + for top in sorted(running): + failed = True + out.write(f"--- {top} did not finish; its output follows\n") + out.write("".join(held.get(top, []))) + out.flush() + + with open(sys.argv[1], "w", encoding="utf-8") as handle: + if package_seconds is not None: + handle.write(f"# package-seconds {package_seconds:.2f}\n") + for name, value in seconds.items(): + handle.write(f"{name} {value:.2f}\n") + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/refresh-internal-api-test-seconds.py b/.github/scripts/refresh-internal-api-test-seconds.py new file mode 100755 index 0000000000..b4f1683f90 --- /dev/null +++ b/.github/scripts/refresh-internal-api-test-seconds.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Rebuild .github/scripts/internal-api-test-seconds.txt from CI timings. + +Every `Backend tests (api-N)` job of Build and Test uploads an +`internal-api-test-seconds-N` artifact written by +record-internal-api-test-seconds.py, holding the elapsed seconds of each +top-level internal/api test it ran under -race on the GitHub runner. This +script downloads those artifacts for one or more runs (or reads files already +on disk), takes the median seconds of every test across the runs, and writes +the weights file that select-internal-api-shard.sh uses to cut the shards. + +Tests at or above --min-seconds are listed by name. Every other test, and any +test added later, weighs the file's DEFAULT_WEIGHT, which is set to the mean +of the unlisted tests so the per-shard totals stay close to what CI measured. + +Usage, from the repository root: + python3 .github/scripts/refresh-internal-api-test-seconds.py --run [--run ...] + python3 .github/scripts/refresh-internal-api-test-seconds.py --dir + +Use runs whose api shards all passed, ideally two or three recent ones, since +a single runner can be 30% faster or slower than the next. Pass --tests with +the output of `go test -list . ./internal/api` to also print the shard cut +the new weights produce. +""" + +from __future__ import annotations + +import argparse +import datetime +import statistics +import subprocess +import sys +import tempfile +from collections import defaultdict +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +WEIGHTS = REPO / ".github" / "scripts" / "internal-api-test-seconds.txt" +SELECTOR = REPO / ".github" / "scripts" / "select-internal-api-shard.sh" +ARTIFACT_PATTERN = "internal-api-test-seconds-*" + + +def download(run_id: str, into: Path, repo: str) -> None: + subprocess.run( + ["gh", "run", "download", run_id, "--repo", repo, "--pattern", ARTIFACT_PATTERN, "--dir", str(into)], + check=True, + ) + + +def read_timings(files: list[Path]) -> tuple[dict[str, list[float]], list[tuple[Path, float | None, float, int]]]: + samples: dict[str, list[float]] = defaultdict(list) + shards = [] + for path in files: + package_seconds = None + total = 0.0 + count = 0 + for line in path.read_text(encoding="utf-8").splitlines(): + fields = line.split() + if len(fields) == 3 and fields[:2] == ["#", "package-seconds"]: + package_seconds = float(fields[2]) + continue + if len(fields) != 2 or fields[0].startswith("#"): + continue + seconds = float(fields[1]) + samples[fields[0]].append(seconds) + total += seconds + count += 1 + shards.append((path, package_seconds, total, count)) + return samples, shards + + +def render(weights: dict[str, float], default: float, sources: list[str], min_seconds: float) -> str: + today = datetime.date.today().isoformat() + header = f"""\ +# Run time in seconds of the slowest internal/api top-level tests under -race +# on a GitHub ubuntu-24.04 runner, the median of the Build and Test runs named +# below. Used only by select-internal-api-shard.sh to choose where the +# Backend tests (api-N) shards cut go test's list. Tests not named here weigh +# DEFAULT_WEIGHT, the mean of the tests under {min_seconds:g}s in the same runs. A +# missing, stale or renamed entry can make the shards uneven but can never +# drop or repeat a test. +# +# Generated {today} by .github/scripts/refresh-internal-api-test-seconds.py +# from {', '.join(sources)}. Every api shard uploads an +# internal-api-test-seconds-N artifact, so rerun that script with recent +# passing runs when one api shard runs well over the others. +DEFAULT_WEIGHT {default:.2f} +""" + body = "".join(f"{name} {seconds:.2f}\n" for name, seconds in weights.items()) + return header + body + + +def report_cut(weights_path: Path, tests_path: Path, shards: int, samples: dict[str, float], default: float) -> None: + tests = [line.strip() for line in tests_path.read_text(encoding="utf-8").splitlines()] + tests = [t for t in tests if t.startswith(("Test", "Fuzz", "Example"))] + stdin = "\n".join(tests) + "\n" + print(f"Predicted cut with {shards} shards (measured seconds, default {default:.2f}s when unmeasured):") + for index in range(shards): + selected = subprocess.run( + ["bash", str(SELECTOR), str(weights_path), str(shards), str(index)], + input=stdin, capture_output=True, text=True, check=True, + ).stdout.split() + seconds = sum(samples.get(name, default) for name in selected) + print(f" api-{index}: {len(selected):5d} tests {seconds:7.1f}s {selected[0]} .. {selected[-1]}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--run", action="append", default=[], help="Build and Test run id to download artifacts from") + parser.add_argument("--dir", action="append", default=[], type=Path, help="directory of downloaded artifacts") + parser.add_argument("--repo", default="rcourtman/Pulse") + parser.add_argument("--min-seconds", type=float, default=0.5) + parser.add_argument("--output", type=Path, default=WEIGHTS) + parser.add_argument("--tests", type=Path, help="go test -list output, to print the resulting shard cut") + parser.add_argument("--shards", type=int, help="shard count for --tests (default: API_SHARD_COUNT from the workflow)") + args = parser.parse_args() + if not args.run and not args.dir: + parser.error("pass at least one --run or --dir") + + with tempfile.TemporaryDirectory() as scratch: + dirs = list(args.dir) + for run_id in args.run: + target = Path(scratch) / run_id + download(run_id, target, args.repo) + dirs.append(target) + files = sorted(path for d in dirs for path in d.rglob("*.txt")) + if not files: + print("no timing files found", file=sys.stderr) + return 1 + by_test, shards = read_timings(files) + + for path, package_seconds, total, count in shards: + wall = f"{package_seconds:7.1f}s" if package_seconds is not None else " ?" + print(f"{path.parent.name}/{path.name}: {count} tests, {total:.1f}s in tests, package {wall}") + + median = {name: statistics.median(values) for name, values in by_test.items()} + listed = {name: seconds for name, seconds in median.items() if seconds >= args.min_seconds} + rest = [seconds for name, seconds in median.items() if name not in listed] + default = max(statistics.fmean(rest) if rest else 0.05, 0.01) + + sources = [f"run {run_id}" for run_id in args.run] + [str(d) for d in args.dir] + args.output.write_text(render(listed, default, sources, args.min_seconds), encoding="utf-8") + print(f"Wrote {len(listed)} tests at {args.min_seconds:g}s or more and DEFAULT_WEIGHT {default:.2f} to {args.output}") + + if args.tests: + shards_count = args.shards + if shards_count is None: + workflow = (REPO / ".github" / "workflows" / "build-and-test.yml").read_text(encoding="utf-8") + shards_count = int(workflow.split("API_SHARD_COUNT: ", 1)[1].split()[0]) + report_cut(args.output, args.tests, shards_count, median, default) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/select-internal-api-shard.sh b/.github/scripts/select-internal-api-shard.sh index 3a0cf0cd4f..512d064383 100755 --- a/.github/scripts/select-internal-api-shard.sh +++ b/.github/scripts/select-internal-api-shard.sh @@ -7,8 +7,10 @@ # Shards are contiguous runs of go test's order, because some internal/api # tests still depend on package state left by the tests just before them. # Cut points are chosen by estimated run time instead of test count. Every -# test weighs its seconds from the weights file, or DEFAULT_WEIGHT when the -# file does not name it (new tests, renamed tests). The smallest per-shard +# test weighs its seconds from the weights file, or the default weight when +# the file does not name it (new tests, renamed tests). The default weight is +# the file's `DEFAULT_WEIGHT ` line, 0.05 when it has none, and a +# DEFAULT_WEIGHT environment variable overrides both. The smallest per-shard # capacity that lets the list be cut into at most contiguous # runs is found by binary search, then the list is filled greedily up to that # capacity. If fewer runs than shards would result, the tail is cut early so @@ -39,7 +41,7 @@ if [ ! -f "$weights" ]; then exit 2 fi -awk -v n="$count" -v want="$index" -v default_weight="${DEFAULT_WEIGHT:-0.05}" ' +awk -v n="$count" -v want="$index" -v env_default="${DEFAULT_WEIGHT:-}" ' function centis(seconds, c) { c = int(seconds * 100 + 0.5) return c < 1 ? 1 : c @@ -64,10 +66,14 @@ awk -v n="$count" -v want="$index" -v default_weight="${DEFAULT_WEIGHT:-0.05}" ' bad = 1 exit 1 } - weight[$1] = $2 + 0 + if ($1 == "DEFAULT_WEIGHT") file_default = $2 + 0 + else weight[$1] = $2 + 0 next } NF { + if (!default_weight) { + default_weight = env_default != "" ? env_default + 0 : (file_default ? file_default : 0.05) + } total++ name[total] = $1 cost[total] = centis(($1 in weight) ? weight[$1] : default_weight) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index cc54b9a6d7..701861305e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -337,9 +337,9 @@ jobs: fail-fast: false matrix: # Keep API_SHARD_COUNT equal to the number of indexes listed here. - index: [0, 1, 2, 3] + index: [0, 1, 2, 3, 4] env: - API_SHARD_COUNT: 4 + API_SHARD_COUNT: 5 steps: - name: Report documentation-only change @@ -387,7 +387,12 @@ jobs: # small default weight, and the weights only move the cut points, so # coverage never depends on them. Listing with -race builds the same # test binary the run then reuses from cache. Benchmarks are left out - # because plain go test never runs them. + # because plain go test never runs them. The run goes through + # -json so every shard records each test's seconds on the runner + # (uploaded below and fed back into the weights by + # refresh-internal-api-test-seconds.py). The recorder prints only + # package lines and the output of failing or unfinished tests, like + # plain go test, and pipefail keeps a go test failure fatal. run: | set -euo pipefail listing=$(go test -race -list . ./internal/api) @@ -406,14 +411,17 @@ jobs: others=$(printf '%s\n' "$tests" | grep -vxF -f <(printf '%s\n' "$selected") || true) # Name whichever side is shorter. A shard holding most of the tests # is passed as -skip of every other shard's tests, which runs the - # same set in the same order. + # same set in the same order. Never pair -skip with -run. The test + # binary caches only the last compiled pattern, so alternating the + # two recompiles the long skip pattern for every test and subtest, + # which added about seven minutes to such a shard. pattern="^($(printf '%s\n' "$selected" | paste -sd '|' -))\$" filter=(-run "$pattern") if [ -n "$others" ]; then skip="^($(printf '%s\n' "$others" | paste -sd '|' -))\$" if [ "${#skip}" -lt "${#pattern}" ]; then pattern=$skip - filter=(-run . -skip "$pattern") + filter=(-skip "$pattern") fi fi # Linux rejects a single argument over 128 KiB. Fail with a clear @@ -423,7 +431,19 @@ jobs: exit 1 fi echo "internal/api shard ${API_SHARD_INDEX} of ${API_SHARD_COUNT} runs $(printf '%s\n' "$selected" | wc -l | tr -d ' ') of ${total} tests (${selected%%$'\n'*} .. ${selected##*$'\n'})" - go test -race -timeout 50m "${filter[@]}" ./internal/api + timings="$RUNNER_TEMP/internal-api-test-seconds" + mkdir -p "$timings" + go test -race -timeout 50m -json "${filter[@]}" ./internal/api \ + | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt" + + - name: Upload internal/api test timings + if: always() && needs.changes.outputs.code == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: internal-api-test-seconds-${{ matrix.index }} + path: ${{ runner.temp }}/internal-api-test-seconds/ + if-no-files-found: warn + retention-days: 30 # Required check name. Branch protection matches `Backend tests (api)` # exactly, so this job must keep that name and must fail unless every diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 1cf07668de..169e32b9b8 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -26,7 +26,7 @@ needs its own checks. Release publication workflows are unaffected. ### Sharded internal/api backend tests -Build and Test runs the `internal/api` race tests as four `Backend tests +Build and Test runs the `internal/api` race tests as five `Backend tests (api-N)` shards instead of one job, because that one package took about 27 of the 31 minutes of the old `Backend tests (api)` job and set the critical path of every pull request. Each shard lists the package's tests with `go test @@ -41,16 +41,35 @@ contract mock chart test runs about six minutes on the runner and the integration server tail about twenty seconds per test. `.github/scripts/select-internal-api-shard.sh` weighs each test by its seconds in `.github/scripts/internal-api-test-seconds.txt` (the slowest tests only, -with a small default for every unlisted, new or renamed test), finds the -smallest per-shard budget that fits the list into the shard count, and fills -contiguous slices up to it, cutting the tail early if needed so no shard is -empty. The weights only move cut points. Each test is assigned one shard in a -single pass, so a stale weights file can unbalance the shards but never drop -or repeat a test, and the file is refreshed from a `-race -json` run when one -shard drifts well past the others. A shard holding most of the tests passes -`-skip` of every other shard's tests instead of naming its own, which runs the -same tests in the same order and keeps the single argument under the 120,000 -byte ceiling. The required check name +plus a `DEFAULT_WEIGHT` line for every unlisted, new or renamed test), finds +the smallest per-shard budget that fits the list into the shard count, and +fills contiguous slices up to it, cutting the tail early if needed so no shard +is empty. The weights only move cut points. Each test is assigned one shard in +a single pass, so a stale weights file can unbalance the shards but never drop +or repeat a test. The weights must describe the GitHub runner, not a +workstation. A first seeding from one local run scaled by guesswork left the +four shards at 4.5, 8.8, 17.1 and 12.5 minutes. Every shard therefore runs +`go test -json` through `.github/scripts/record-internal-api-test-seconds.py`, +which prints only package lines and the output of failing or unfinished tests +(as plain `go test` does), exits non-zero on any failure behind `pipefail`, +and writes each top-level test's seconds to an `internal-api-test-seconds-N` +artifact that is uploaded even when the shard fails. +`.github/scripts/refresh-internal-api-test-seconds.py --run ` downloads +those artifacts with `gh run download`, takes each test's median across the +given runs, and rewrites the weights file with `DEFAULT_WEIGHT` set to the mean +of the unlisted tests. Until such runs exist, the weights are a local +`-race -json` run with each region of the list scaled to the seconds the +runner reported for it across the count-based and weighted runs. Five shards +put the six minute contract mock chart test with only its next few +neighbours, and the predicted slowest shard is about 8.5 minutes against about +10 with four, which keeps a 20% slower runner under the 11 minutes of the +Frontend and rest checks. A shard holding most of the +tests passes `-skip` of every other shard's tests instead of naming its own, +which runs the same tests in the same order and keeps the single argument +under the 120,000 byte ceiling. It passes `-skip` alone. The test binary +caches only its last compiled pattern, so `-run .` next to `-skip` recompiled +the long skip pattern for every test and subtest and roughly doubled that +shard. The required check name `Backend tests (api)` belongs to a verdict job that passes only when every shard succeeded, and `Backend tests (rest-0)` and `Backend tests (rest-1)` keep their names and package split. All shards still expand for documentation-only diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index fc356b735b..8df54c6569 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4512,7 +4512,9 @@ ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", ".github/scripts/internal-api-test-seconds.txt", + ".github/scripts/record-internal-api-test-seconds.py", ".github/scripts/recover-demo-runtime.sh", + ".github/scripts/refresh-internal-api-test-seconds.py", ".github/scripts/resolve-demo-runtime-profile.sh", ".github/scripts/select-internal-api-shard.sh", ".github/scripts/setup-demo-ssh.sh", @@ -5038,6 +5040,8 @@ "match_prefixes": [], "match_files": [ ".github/scripts/internal-api-test-seconds.txt", + ".github/scripts/record-internal-api-test-seconds.py", + ".github/scripts/refresh-internal-api-test-seconds.py", ".github/scripts/select-internal-api-shard.sh", ".github/workflows/build-and-test.yml", "frontend-modern/package-lock.json", diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index bb72037e38..8f89371cc1 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -3848,15 +3848,28 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) { // A shard holding most tests is named by skipping every other // shard's tests, which keeps its argument under the exec limit. `others=$(printf '%s\n' "$tests" | grep -vxF -f <(printf '%s\n' "$selected") || true)`, - `filter=(-run . -skip "$pattern")`, + `filter=(-skip "$pattern")`, `if [ "${#pattern}" -gt 120000 ]; then`, - `go test -race -timeout 50m "${filter[@]}" ./internal/api`, + // Every shard records per-test seconds on the runner through -json + // so the weights can be refreshed from CI, while pipefail keeps a + // go test failure fatal behind the recorder. + "set -euo pipefail", + `go test -race -timeout 50m -json "${filter[@]}" ./internal/api \ + | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt"`, + "if: always() && needs.changes.outputs.code == 'true'", + "name: internal-api-test-seconds-${{ matrix.index }}", + "path: ${{ runner.temp }}/internal-api-test-seconds/", "PULSE_DATA_DIR: /tmp/pulse-test-data", } { if !strings.Contains(shards, required) { t.Fatalf("internal/api shard job missing %q", required) } } + // The test binary caches one compiled pattern, so -run next to -skip + // recompiles the long skip pattern for every test and subtest. + if strings.Contains(shards, "-run . -skip") { + t.Fatal("internal/api shards must pass -skip alone; pairing it with -run recompiles the skip pattern per test") + } if strings.Contains(shards, "sort") { t.Fatal("internal/api shards must keep go test's run order; sorting splits order-coupled tests") } @@ -3881,6 +3894,67 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) { } assertInternalAPIShardSelectionExhaustive(t, want) + assertInternalAPITimingRecorderKeepsFailuresVisible(t) +} + +// assertInternalAPITimingRecorderKeepsFailuresVisible feeds the -json +// recorder a passing, a failing and an unfinished test. Passing output must +// stay hidden like plain go test, failing and unfinished output must print, +// any failure must exit non-zero, and every finished top-level test must be +// written with its seconds. +func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) { + t.Helper() + recorder := repoFile(".github", "scripts", "record-internal-api-test-seconds.py") + record := func(events string) (string, string, error) { + out := filepath.Join(t.TempDir(), "seconds.txt") + cmd := exec.Command("python3", recorder, out) + cmd.Stdin = strings.NewReader(events) + printed, err := cmd.Output() + written, readErr := os.ReadFile(out) + if readErr != nil { + t.Fatalf("recorder wrote no seconds file: %v", readErr) + } + return string(printed), string(written), err + } + const pkg = `"Package":"example/internal/api"` + passing := strings.Join([]string{ + `{"Action":"run",` + pkg + `,"Test":"TestQuiet"}`, + `{"Action":"output",` + pkg + `,"Test":"TestQuiet","Output":"quiet log line\n"}`, + `{"Action":"pass",` + pkg + `,"Test":"TestQuiet","Elapsed":1.25}`, + `{"Action":"output",` + pkg + `,"Output":"ok \texample/internal/api\t2.000s\n"}`, + `{"Action":"pass",` + pkg + `,"Elapsed":2}`, + }, "\n") + "\n" + printed, written, err := record(passing) + if err != nil { + t.Fatalf("recorder must pass a passing run: %v", err) + } + if strings.Contains(printed, "quiet log line") || !strings.Contains(printed, "ok \texample/internal/api") { + t.Fatalf("recorder must print package lines and hide passing test output, printed %q", printed) + } + if written != "# package-seconds 2.00\nTestQuiet 1.25\n" { + t.Fatalf("recorder seconds file = %q", written) + } + + failing := strings.Join([]string{ + `{"Action":"run",` + pkg + `,"Test":"TestBroken"}`, + `{"Action":"output",` + pkg + `,"Test":"TestBroken/case","Output":"broken detail\n"}`, + `{"Action":"fail",` + pkg + `,"Test":"TestBroken/case","Elapsed":0.5}`, + `{"Action":"fail",` + pkg + `,"Test":"TestBroken","Elapsed":0.75}`, + `{"Action":"run",` + pkg + `,"Test":"TestHung"}`, + `{"Action":"output",` + pkg + `,"Test":"TestHung","Output":"panic: test timed out\n"}`, + }, "\n") + "\n" + printed, written, err = record(failing) + if err == nil { + t.Fatal("recorder must exit non-zero when a test fails or never finishes") + } + for _, want := range []string{"broken detail", "TestHung did not finish", "panic: test timed out"} { + if !strings.Contains(printed, want) { + t.Fatalf("recorder must print %q for failing or unfinished tests, printed %q", want, printed) + } + } + if written != "TestBroken 0.75\n" { + t.Fatalf("recorder must record only finished top-level tests, wrote %q", written) + } } // assertInternalAPIShardSelectionExhaustive runs the shard selector the way @@ -3904,12 +3978,15 @@ func assertInternalAPIShardSelectionExhaustive(t *testing.T, workflowShards int) if len(fields) == 0 || strings.HasPrefix(fields[0], "#") { continue } - if len(fields) != 2 || !strings.HasPrefix(fields[0], "Test") { - t.Fatalf("internal/api weights line must be ` `, got %q", line) + if len(fields) != 2 || !(strings.HasPrefix(fields[0], "Test") || fields[0] == "DEFAULT_WEIGHT") { + t.Fatalf("internal/api weights line must be ` ` or `DEFAULT_WEIGHT `, got %q", line) } if seconds, err := strconv.ParseFloat(fields[1], 64); err != nil || seconds <= 0 { t.Fatalf("internal/api weight for %s must be positive seconds, got %q", fields[0], fields[1]) } + if fields[0] == "DEFAULT_WEIGHT" { + continue + } weighted = append(weighted, fields[0]) } if len(weighted) == 0 { @@ -3928,7 +4005,7 @@ func assertInternalAPIShardSelectionExhaustive(t *testing.T, workflowShards int) t.Fatal(err) } skewedWeights := filepath.Join(t.TempDir(), "skewed.txt") - skewed := tests[len(tests)/3] + " 900\n" + tests[len(tests)-1] + " 450\n" + skewed := "DEFAULT_WEIGHT 7.5\n" + tests[len(tests)/3] + " 900\n" + tests[len(tests)-1] + " 450\n" if err := os.WriteFile(skewedWeights, []byte(skewed), 0o644); err != nil { t.Fatal(err) }