120 lines
2.9 KiB
Go
120 lines
2.9 KiB
Go
// Package resilience 服务调用的三件套：超时（由调用方的 context 控制）、带退避的重试、熔断器。只用标准库。
|
|
package resilience
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"math/rand/v2"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// ErrOpen 熔断器打开时直接返回，不再调用下游
|
|
var ErrOpen = errors.New("熔断中：下游暂时不可用")
|
|
|
|
// Permanent 包装不应该重试的错误（参数错误、没有权限、资源不存在等）
|
|
type Permanent struct{ Err error }
|
|
|
|
func (p Permanent) Error() string { return p.Err.Error() }
|
|
func (p Permanent) Unwrap() error { return p.Err }
|
|
|
|
// Retry 最多尝试 attempts 次，间隔指数增长并加随机抖动；context 取消或超时立即停止
|
|
func Retry(ctx context.Context, attempts int, base time.Duration, fn func(context.Context) error) error {
|
|
var err error
|
|
for i := range attempts {
|
|
if err = fn(ctx); err == nil {
|
|
return nil
|
|
}
|
|
var p Permanent
|
|
if errors.As(err, &p) || errors.Is(err, ErrOpen) || i == attempts-1 {
|
|
return err
|
|
}
|
|
backoff := base << i // base, 2*base, 4*base ...
|
|
backoff += time.Duration(rand.Int64N(int64(backoff)/2 + 1)) // 抖动，避免所有客户端同时重试
|
|
select {
|
|
case <-ctx.Done():
|
|
return errors.Join(err, ctx.Err())
|
|
case <-time.After(backoff):
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
type state int
|
|
|
|
const (
|
|
closed state = iota // 正常放行
|
|
open // 熔断：直接拒绝
|
|
halfOpen // 试探：放一个请求过去
|
|
)
|
|
|
|
// Breaker 连续失败 threshold 次后打开，cooldown 之后进入半开，试探成功则关闭、失败则重新打开
|
|
type Breaker struct {
|
|
mu sync.Mutex
|
|
state state
|
|
failures int
|
|
threshold int
|
|
cooldown time.Duration
|
|
openedAt time.Time
|
|
probing bool
|
|
now func() time.Time // 测试时可以替换
|
|
}
|
|
|
|
func NewBreaker(threshold int, cooldown time.Duration) *Breaker {
|
|
return &Breaker{threshold: threshold, cooldown: cooldown, now: time.Now}
|
|
}
|
|
|
|
func (b *Breaker) allow() bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
switch b.state {
|
|
case open:
|
|
if b.now().Sub(b.openedAt) < b.cooldown {
|
|
return false
|
|
}
|
|
b.state = halfOpen
|
|
fallthrough
|
|
case halfOpen:
|
|
if b.probing {
|
|
return false // 半开状态只放一个试探请求
|
|
}
|
|
b.probing = true
|
|
return true
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (b *Breaker) record(err error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
var p Permanent
|
|
failed := err != nil && !errors.As(err, &p) // 业务错误不算下游故障
|
|
if b.state == halfOpen {
|
|
b.probing = false
|
|
if failed {
|
|
b.state, b.openedAt = open, b.now()
|
|
} else {
|
|
b.state, b.failures = closed, 0
|
|
}
|
|
return
|
|
}
|
|
if !failed {
|
|
b.failures = 0
|
|
return
|
|
}
|
|
b.failures++
|
|
if b.failures >= b.threshold {
|
|
b.state, b.openedAt = open, b.now()
|
|
}
|
|
}
|
|
|
|
// Do 通过熔断器调用 fn
|
|
func (b *Breaker) Do(ctx context.Context, fn func(context.Context) error) error {
|
|
if !b.allow() {
|
|
return ErrOpen
|
|
}
|
|
err := fn(ctx)
|
|
b.record(err)
|
|
return err
|
|
}
|