上一篇文章分析了ContainerGC,k8s為了回收機(jī)器上的存儲(chǔ)資源同時(shí)有ImageGC對(duì)image資源進(jìn)行回收蜡塌。
ImageGC同樣定義了image gc manager和gc policy央勒。
//pkg/kubelet/image/image_gc_manager.go
type ImageGCPolicy struct {
// Any usage above this threshold will always trigger garbage collection.
// This is the highest usage we will allow.
HighThresholdPercent int
// Any usage below this threshold will never trigger garbage collection.
// This is the lowest threshold we will try to garbage collect to.
LowThresholdPercent int
// Minimum age at which an image can be garbage collected.
MinAge time.Duration
}
type realImageGCManager struct {
// Container runtime
runtime container.Runtime
// Records of images and their use.
imageRecords map[string]*imageRecord
imageRecordsLock sync.Mutex
// The image garbage collection policy in use.
policy ImageGCPolicy
// cAdvisor instance.
cadvisor cadvisor.Interface
// Recorder for Kubernetes events.
recorder record.EventRecorder
// Reference to this node.
nodeRef *v1.ObjectReference
// Track initialization
initialized bool
// imageCache is the cache of latest image list.
imageCache imageCache
}
策略也主要有三個(gè)參數(shù):
- HighThresholdPercent 高于此閾值將進(jìn)行回收
- LowThresholdPercent 低于此閾值將不會(huì)觸發(fā)回收
- MinAge 回收image的最小年齡
在這個(gè)文件里同樣有一個(gè)GarbageCollect方法
func (im *realImageGCManager) detectImages(detectTime time.Time) error {
images, err := im.runtime.ListImages()
if err != nil {
return err
}
pods, err := im.runtime.GetPods(true)
if err != nil {
return err
}
// Make a set of images in use by containers.
imagesInUse := sets.NewString()
for _, pod := range pods {
for _, container := range pod.Containers {
glog.V(5).Infof("Pod %s/%s, container %s uses image %s(%s)", pod.Namespace, pod.Name, container.Name, container.Image, container.ImageID)
imagesInUse.Insert(container.ImageID)
}
}
// Add new images and record those being used.
now := time.Now()
currentImages := sets.NewString()
im.imageRecordsLock.Lock()
defer im.imageRecordsLock.Unlock()
for _, image := range images {
glog.V(5).Infof("Adding image ID %s to currentImages", image.ID)
currentImages.Insert(image.ID)
// New image, set it as detected now.
if _, ok := im.imageRecords[image.ID]; !ok {
glog.V(5).Infof("Image ID %s is new", image.ID)
im.imageRecords[image.ID] = &imageRecord{
firstDetected: detectTime,
}
}
// Set last used time to now if the image is being used.
if isImageUsed(image, imagesInUse) {
glog.V(5).Infof("Setting Image ID %s lastUsed to %v", image.ID, now)
im.imageRecords[image.ID].lastUsed = now
}
glog.V(5).Infof("Image ID %s has size %d", image.ID, image.Size)
im.imageRecords[image.ID].size = image.Size
}
// Remove old images from our records.
for image := range im.imageRecords {
if !currentImages.Has(image) {
glog.V(5).Infof("Image ID %s is no longer present; removing from imageRecords", image)
delete(im.imageRecords, image)
}
}
return nil
}
func (im *realImageGCManager) GarbageCollect() error {
// Get disk usage on disk holding images.
fsInfo, err := im.cadvisor.ImagesFsInfo()
if err != nil {
return err
}
capacity := int64(fsInfo.Capacity)
available := int64(fsInfo.Available)
if available > capacity {
glog.Warningf("available %d is larger than capacity %d", available, capacity)
available = capacity
}
// Check valid capacity.
if capacity == 0 {
err := fmt.Errorf("invalid capacity %d on device %q at mount point %q", capacity, fsInfo.Device, fsInfo.Mountpoint)
im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.InvalidDiskCapacity, err.Error())
return err
}
// If over the max threshold, free enough to place us at the lower threshold.
usagePercent := 100 - int(available*100/capacity)
if usagePercent >= im.policy.HighThresholdPercent {
amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available
glog.Infof("[imageGCManager]: Disk usage on %q (%s) is at %d%% which is over the high threshold (%d%%). Trying to free %d bytes", fsInfo.Device, fsInfo.Mountpoint, usagePercent, im.policy.HighThresholdPercent, amountToFree)
freed, err := im.freeSpace(amountToFree, time.Now())
if err != nil {
return err
}
if freed < amountToFree {
err := fmt.Errorf("failed to garbage collect required amount of images. Wanted to free %d bytes, but freed %d bytes", amountToFree, freed)
im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.FreeDiskSpaceFailed, err.Error())
return err
}
}
return nil
}
func (im *realImageGCManager) DeleteUnusedImages() (int64, error) {
return im.freeSpace(math.MaxInt64, time.Now())
}
// Tries to free bytesToFree worth of images on the disk.
//
// Returns the number of bytes free and an error if any occurred. The number of
// bytes freed is always returned.
// Note that error may be nil and the number of bytes free may be less
// than bytesToFree.
func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) (int64, error) {
err := im.detectImages(freeTime)
if err != nil {
return 0, err
}
im.imageRecordsLock.Lock()
defer im.imageRecordsLock.Unlock()
// Get all images in eviction order.
images := make([]evictionInfo, 0, len(im.imageRecords))
for image, record := range im.imageRecords {
images = append(images, evictionInfo{
id: image,
imageRecord: *record,
})
}
sort.Sort(byLastUsedAndDetected(images))
// Delete unused images until we've freed up enough space.
var deletionErrors []error
spaceFreed := int64(0)
for _, image := range images {
glog.V(5).Infof("Evaluating image ID %s for possible garbage collection", image.id)
// Images that are currently in used were given a newer lastUsed.
if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) {
glog.V(5).Infof("Image ID %s has lastUsed=%v which is >= freeTime=%v, not eligible for garbage collection", image.id, image.lastUsed, freeTime)
break
}
// Avoid garbage collect the image if the image is not old enough.
// In such a case, the image may have just been pulled down, and will be used by a container right away.
if freeTime.Sub(image.firstDetected) < im.policy.MinAge {
glog.V(5).Infof("Image ID %s has age %v which is less than the policy's minAge of %v, not eligible for garbage collection", image.id, freeTime.Sub(image.firstDetected), im.policy.MinAge)
continue
}
// Remove image. Continue despite errors.
glog.Infof("[imageGCManager]: Removing image %q to free %d bytes", image.id, image.size)
err := im.runtime.RemoveImage(container.ImageSpec{Image: image.id})
if err != nil {
deletionErrors = append(deletionErrors, err)
continue
}
delete(im.imageRecords, image.id)
spaceFreed += image.size
if spaceFreed >= bytesToFree {
break
}
}
if len(deletionErrors) > 0 {
return spaceFreed, fmt.Errorf("wanted to free %d bytes, but freed %d bytes space with errors in image deletion: %v", bytesToFree, spaceFreed, errors.NewAggregate(deletionErrors))
}
return spaceFreed, nil
}
GarbageCollect方法也是在pkg/kubelet/kubelet.go里面進(jìn)行調(diào)用勺美，每調(diào)用一次執(zhí)行一次回收流程比庄，每次調(diào)用間隔是5分鐘叙谨。
GarbageCollect首先會(huì)調(diào)用cadvisor獲取監(jiān)控機(jī)器上的資源總量可用量审轮，當(dāng)發(fā)現(xiàn)使用的資源已經(jīng)大于設(shè)置的最高閾值將會(huì)觸發(fā)回收行為损敷，并且計(jì)算出需要回收的空間大小烂斋。
freeSpace方法則是真正的回收過(guò)程屹逛，過(guò)程分兩步。
- 探測(cè)機(jī)器上的image
- 刪除老的image
探測(cè)鏡像:
探測(cè)機(jī)器上的image是獲取機(jī)器上的image列表并如果是正在使用的image標(biāo)記最后使用時(shí)間和image大小和第一次探測(cè)到的時(shí)間汛骂，并把這個(gè)列表的image放到imageRecords里面進(jìn)行緩存罕模，imageRecords是個(gè)map結(jié)結(jié)構(gòu)的數(shù)據(jù)類型。
回收鏡像：
回收鏡像首先遍歷imageRecords里面的鏡像帘瞭，放到一個(gè)數(shù)組里面排序淑掌，按照最后使用時(shí)間或者第一次探測(cè)到的時(shí)間進(jìn)行排序。
下一步則對(duì)數(shù)組里面的鏡像進(jìn)行回收蝶念，如果鏡像還在使用中或者最后探測(cè)到的時(shí)間小于設(shè)置的MinAge則不進(jìn)行回收抛腕，否則刪除鏡像∶窖常回收過(guò)程中如果發(fā)現(xiàn)現(xiàn)在機(jī)器上的資源已經(jīng)小于設(shè)置的LowThresholdPercent那么跳出回收流程担敌。
