mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 02:55:51 +00:00
Honor the configured AI timeout while waiting for the first stream chunk
The OpenAI-compatible stream reader bounded every chunk wait at 12s. Local backends (LM Studio, llama.cpp) legitimately spend minutes on prompt processing before the first SSE chunk, so raising the provider timeout in Settings changed nothing and Pulse dropped the stream with 'AI response timed out' (discussion #1571). The wait for first bytes now uses the configured request timeout; the 12s bound still applies to inter-chunk gaps once the stream is flowing.
This commit is contained in:
@@ -48,6 +48,11 @@ type OpenAIClient struct {
|
||||
// body chunks. Bound that gap separately from the full turn timeout so chat
|
||||
// fallback can move before the drawer looks dead.
|
||||
streamChunkTimeout time.Duration
|
||||
// Local OpenAI-compatible backends (LM Studio, llama.cpp, vLLM on CPU) can
|
||||
// legitimately spend minutes on prompt processing before the first SSE
|
||||
// chunk, so the wait for first bytes honors the configured request timeout
|
||||
// instead of the inter-chunk gap bound (issue discussion #1571).
|
||||
streamFirstChunkTimeout time.Duration
|
||||
}
|
||||
|
||||
// NewOpenAIClient creates a new OpenAI API client
|
||||
@@ -72,14 +77,15 @@ func NewOpenAICompatibleClient(providerName, apiKey, model, baseURL string, time
|
||||
timeout = 300 * time.Second // Default 5 minutes
|
||||
}
|
||||
return &OpenAIClient{
|
||||
providerName: providerName,
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
baseURL: baseURL,
|
||||
requestTimeout: timeout,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
streamClient: newOpenAIStreamHTTPClient(timeout),
|
||||
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
|
||||
providerName: providerName,
|
||||
apiKey: apiKey,
|
||||
model: model,
|
||||
baseURL: baseURL,
|
||||
requestTimeout: timeout,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
streamClient: newOpenAIStreamHTTPClient(timeout),
|
||||
streamChunkTimeout: boundedOpenAIStreamChunkTimeout(timeout),
|
||||
streamFirstChunkTimeout: timeout,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1126,9 +1132,15 @@ func (c *OpenAIClient) ChatStream(ctx context.Context, req ChatRequest, callback
|
||||
return false, nil
|
||||
}
|
||||
|
||||
receivedFirstChunk := false
|
||||
for {
|
||||
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, c.streamChunkTimeout)
|
||||
chunkTimeout := c.streamChunkTimeout
|
||||
if !receivedFirstChunk {
|
||||
chunkTimeout = c.streamFirstChunkTimeout
|
||||
}
|
||||
n, err := readOpenAIStreamChunk(streamCtx, reader, buf, chunkTimeout)
|
||||
if n > 0 {
|
||||
receivedFirstChunk = true
|
||||
pendingData += string(buf[:n])
|
||||
lines := strings.Split(pendingData, "\n")
|
||||
|
||||
|
||||
@@ -187,12 +187,14 @@ func TestNewOpenAIClient_BoundsStreamResponseHeaderTimeout(t *testing.T) {
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, openaiStreamResponseHeaderTimeout, transport.ResponseHeaderTimeout)
|
||||
assert.Equal(t, openaiStreamChunkTimeout, client.streamChunkTimeout)
|
||||
assert.Equal(t, 300*time.Second, client.streamFirstChunkTimeout)
|
||||
|
||||
shortTimeoutClient := NewOpenAIClient("sk-test", "gpt-4", "https://api.openai.com/v1", 2*time.Second)
|
||||
shortTransport, ok := shortTimeoutClient.streamClient.Transport.(*http.Transport)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, 2*time.Second, shortTransport.ResponseHeaderTimeout)
|
||||
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamChunkTimeout)
|
||||
assert.Equal(t, 2*time.Second, shortTimeoutClient.streamFirstChunkTimeout)
|
||||
}
|
||||
|
||||
func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
|
||||
@@ -247,7 +249,9 @@ func TestNewOpenAICompatibleClient_NormalizesProviderBasePaths(t *testing.T) {
|
||||
func TestOpenAIClient_ChatStream_TimesOutWaitingForFirstStreamChunk(t *testing.T) {
|
||||
body := newBlockingReadCloser()
|
||||
client := NewOpenAIClient("sk-test", "gpt-4", "https://example.invalid/v1", time.Second)
|
||||
client.streamChunkTimeout = 10 * time.Millisecond
|
||||
// The wait for first bytes honors the full request timeout (local backends
|
||||
// can spend minutes on prompt processing), not the inter-chunk gap bound.
|
||||
client.streamFirstChunkTimeout = 10 * time.Millisecond
|
||||
client.streamClient = &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
|
||||
Reference in New Issue
Block a user