背景
Today 3:44 PM 有同事 反饋 k8s 拉取鏡像耗時很久溢吻，如下圖所示：
從 log 可以看出桩皿，拉取鏡像花費 2m8s撵渡，但是發(fā)起 Pulling 到 成功 pulled 鏡像中間間隔 42min勾哩，原因何在岔霸？ 后面同事提供完整 log 截圖后發(fā)現(xiàn) waiting 時間就有 42m39s
代碼分析
由于生產(chǎn)環(huán)境中的 k8s 版本為 v1.23.17, 因此我們基于此分支代碼進行分析，進而尋求解決方案犬庇。
// EnsureImageExists pulls the image for the specified pod and container, and returns
// (imageRef, error message, error).
func (m *imageManager) EnsureImageExists(pod *v1.Pod, container *v1.Container, pullSecrets []v1.Secret, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, string, error) {
..........
m.logIt(ref, v1.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("Pulling image %q", container.Image), klog.Info)
startTime := time.Now()
pullChan := make(chan pullResult)
m.puller.pullImage(spec, pullSecrets, pullChan, podSandboxConfig)
imagePullResult := <-pullChan
if imagePullResult.err != nil {
m.logIt(ref, v1.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, imagePullResult.err), klog.Warning)
m.backOff.Next(backOffKey, m.backOff.Clock.Now())
if imagePullResult.err == ErrRegistryUnavailable {
msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image)
return "", msg, imagePullResult.err
}
return "", imagePullResult.err.Error(), ErrImagePull
}
m.logIt(ref, v1.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q in %v (%v including waiting)", container.Image, imagePullResult.pullDuration, time.Since(startTime)), klog.Info)
m.backOff.GC()
return imagePullResult.imageRef, "", nil
}
從代碼（第22行）可以看出僧界，從開啟 pulling 到 pulled 結(jié)束一共花費 42m39s，實際鏡像拉取時間為 2m8s臭挽，因此可以排除 Harbor 的原因捂襟。下面從 ImageManager 的初始化開始分析拉取鏡像的流程。
// NewImageManager instantiates a new ImageManager object.
func NewImageManager(recorder record.EventRecorder, imageService kubecontainer.ImageService, imageBackOff *flowcontrol.Backoff, serialized bool, qps float32, burst int) ImageManager {
imageService = throttleImagePulling(imageService, qps, burst)
var puller imagePuller
if serialized {
puller = newSerialImagePuller(imageService)
} else {
puller = newParallelImagePuller(imageService) puller = newParallelImagePuller(imageService)
}
return &imageManager{
recorder: recorder,
imageService: imageService,
backOff: imageBackOff,
puller: puller,
}
}
從上面代碼可以看出欢峰，初始化 ImageManager時通過指定 serialized 參數(shù)來決定是否是序列化拉取還是并發(fā)拉仍岷伞（其實并發(fā)拉取并未正在實現(xiàn)，只是簡單的起了一個 goroutine 來拉取鏡像纽帖，并沒有做并發(fā)限制闯狱，因此，如果同時拉取鏡像太多會對節(jié)點造成很大壓力）抛计，這個參數(shù)是由 kubelet 的 serializeImagePulls 來控制的，而/var/lib/kubelet/config.yaml 中 serializeImagePulls 默認值為 true照筑。
serializeImagePulls: true
因此吹截，我們只關心 newSerialImagePuller 的實現(xiàn)過程。
// Maximum number of image pull requests than can be queued.
const maxImagePullRequests = 10
type serialImagePuller struct {
imageService kubecontainer.ImageService
pullRequests chan *imagePullRequest
}
func newSerialImagePuller(imageService kubecontainer.ImageService) imagePuller {
imagePuller := &serialImagePuller{imageService, make(chan *imagePullRequest, maxImagePullRequests)}
go wait.Until(imagePuller.processImagePullRequests, time.Second, wait.NeverStop)
return imagePuller
}
type imagePullRequest struct {
spec kubecontainer.ImageSpec
pullSecrets []v1.Secret
pullChan chan<- pullResult
podSandboxConfig *runtimeapi.PodSandboxConfig
}
func (sip *serialImagePuller) pullImage(spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) {
sip.pullRequests <- &imagePullRequest{
spec: spec,
pullSecrets: pullSecrets,
pullChan: pullChan,
podSandboxConfig: podSandboxConfig,
}
}
func (sip *serialImagePuller) processImagePullRequests() {
for pullRequest := range sip.pullRequests {
startTime := time.Now()
imageRef, err := sip.imageService.PullImage(pullRequest.spec, pullRequest.pullSecrets, pullRequest.podSandboxConfig)
pullRequest.pullChan <- pullResult{
imageRef: imageRef,
err: err,
pullDuration: time.Since(startTime),
}
}
}
serialImagePuller 在初始化時會設置最大拉取鏡像請求數(shù)的隊列凝危，puller 在收到拉取鏡像的請求后會先將此請求放入此隊列波俄，后臺依次從隊列中取出拉取鏡像請求并處理，這樣如果請求數(shù)過多蛾默，或者拉取鏡像比較耗時就會導致后面的拉取鏡像請求一直阻塞懦铺。到這里，就已經(jīng)清楚了為啥 waiting 時間會這么久支鸡。
如何解決
通過查看最新代碼冬念，發(fā)現(xiàn)已經(jīng)實現(xiàn)了并發(fā)拉取，只需要設置以下參數(shù)即可牧挣，其最低支持版本為 v1.27
# Enable parallel image pulls
serializeImagePulls: false
# limit the number of parallel image pulls
maxParallelImagePulls: 10
func (pip *parallelImagePuller) pullImage(ctx context.Context, spec kubecontainer.ImageSpec, pullSecrets []v1.Secret, pullChan chan<- pullResult, podSandboxConfig *runtimeapi.PodSandboxConfig) {
go func() {
if pip.tokens != nil {
pip.tokens <- struct{}{}
defer func() { <-pip.tokens }()
}
startTime := time.Now()
imageRef, err := pip.imageService.PullImage(ctx, spec, pullSecrets, podSandboxConfig)
var size uint64
if err == nil && imageRef != "" {
// Getting the image size with best effort, ignoring the error.
size, _ = pip.imageService.GetImageSize(ctx, spec)
}
pullChan <- pullResult{
imageRef: imageRef,
imageSize: size,
err: err,
pullDuration: time.Since(startTime),
}
}()
}
parallelImagePuller 在拉取鏡像時會先獲取 token急前，相當于控制同時拉取鏡像的并發(fā)數(shù)，只有在獲取到 token 之后才進行鏡像的拉取摆出，以上面設置的值為例稍刀，則支持同時并發(fā)拉取 10 個鏡像，這樣大大緩解 waiting 時間過長的問題俭厚。
此 feature 對應的 enhancement 鏈接為 https://github.com/kubernetes/enhancements/issues/3673
結(jié)論
- 解決方案
升級 k8s 版本
