65 lines
2.0 KiB
Go
65 lines
2.0 KiB
Go
// Package batchfetch 演示企业里常见的“批量调用第三方接口”：
|
|
// 并发上限、单次超时、整体取消、部分失败不影响其他结果、结果顺序与输入一致。
|
|
package batchfetch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type Result struct {
|
|
ID string
|
|
Body string
|
|
Err error
|
|
}
|
|
|
|
// FetchAll 最多同时发出 limit 个请求，每个请求最多等待 perReq。
|
|
func FetchAll(ctx context.Context, client *http.Client, baseURL string, ids []string, limit int, perReq time.Duration) []Result {
|
|
results := make([]Result, len(ids)) // 每个 goroutine 只写自己的下标，不需要加锁
|
|
sem := make(chan struct{}, limit) // 信号量：缓冲区大小就是并发上限
|
|
var wg sync.WaitGroup
|
|
for i, id := range ids {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
select {
|
|
case sem <- struct{}{}: // 拿到名额
|
|
case <-ctx.Done(): // 整体已取消，不再发请求
|
|
results[i] = Result{ID: id, Err: ctx.Err()}
|
|
return
|
|
}
|
|
defer func() { <-sem }()
|
|
results[i] = fetchOne(ctx, client, baseURL, id, perReq)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return results
|
|
}
|
|
|
|
func fetchOne(ctx context.Context, client *http.Client, baseURL, id string, timeout time.Duration) Result {
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/items/"+url.PathEscape(id), nil)
|
|
if err != nil {
|
|
return Result{ID: id, Err: err}
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Result{ID: id, Err: fmt.Errorf("请求 %s: %w", id, err)}
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return Result{ID: id, Err: fmt.Errorf("请求 %s: 状态码 %d", id, resp.StatusCode)}
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) // 限制响应大小，防止对方返回超大内容
|
|
if err != nil {
|
|
return Result{ID: id, Err: fmt.Errorf("读取 %s: %w", id, err)}
|
|
}
|
|
return Result{ID: id, Body: string(body)}
|
|
}
|