526互联

k8s informer源码解析

发布时间 2023-03-28 17:24:12作者: 独揽风月

背景

informer是k8s client-go包里的一个模块，客户端可以通过它来感知事件的变化，而不用直接和apiserver交互，这样减轻了apiserver的负担。

组件介绍

它由以下几个组件组成：

Reflector：

它会采用list/watch的方式获取资源事件，并把它们写入到fifo。

Contoller：

// vendor/k8s.io/client-go/tools/cache/controller.go
func (c *controller) Run(stopCh <-chan struct{}) {
	// ...
	r := NewReflector(
		c.config.ListerWatcher,
		c.config.ObjectType,
		c.config.Queue,
		c.config.FullResyncPeriod,
	)
	// ...
	wg.StartWithChannel(stopCh, r.Run)

	wait.Until(c.processLoop, time.Second, stopCh)
	wg.Wait()
}

1、负责Reflector的执行；

2、从fifo获取事件，先更新indexer缓存，再把事件写入管道。

Processor：

消费管道里的事件，通过事件回调给用户定义的处理函数。

源码分析

package main

import (
	"fmt"
	"k8s.io/client-go/tools/clientcmd"
	"os"
	"time"

	v1 "k8s.io/api/apps/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
)

func main() {
	namespace := "default"
	home, _ := os.UserHomeDir()
	kubeconfig := home + "/.kube/config"
	cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
	clientset, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		panic(err)
	}

	stopCh := make(chan struct{})
	defer close(stopCh)
	shardInformerFactory := informers.NewSharedInformerFactory(clientset, time.Minute)

	deploymentInformer := shardInformerFactory.Apps().V1().Deployments()

	sharedIndexInformer := deploymentInformer.Informer()
	sharedIndexInformer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc:    onAdd,
			UpdateFunc: onUpdate,
			DeleteFunc: onDelete,
		})

	lister := deploymentInformer.Lister()
	shardInformerFactory.Start(stopCh) // 启动informer
	if !cache.WaitForCacheSync(stopCh, sharedIndexInformer.HasSynced) {
		return
	}

	deployments, err := lister.Deployments(namespace).List(labels.Everything())
	if err != nil {
		panic(err)
	}
	for _, deployment := range deployments {
		fmt.Printf("%s\r\n", deployment.Name)
	}
	<-stopCh
}

func onAdd(obj interface{}) {
	deployment := obj.(*v1.Deployment)
	fmt.Printf("onAdd:%s\r\n", deployment.Name)
}

func onUpdate(old, new interface{}) {
	oldDeployment := old.(*v1.Deployment)
	newDeployment := new.(*v1.Deployment)
	fmt.Printf("onUpdate:%s to %s\r\n", oldDeployment.Name, newDeployment.Name)
}

func onDelete(obj interface{}) {
	deployment := obj.(*v1.Deployment)
	fmt.Printf("onDelete:%s\r\n", deployment.Name)
}

deployment已经实现了informer的接口，我们可以以它为例来看看informer是怎么运作的。

1. 初始化 deploymentInformer

shardInformerFactory := informers.NewSharedInformerFactory(clientset, time.Minute)

deploymentInformer := shardInformerFactory.Apps().V1().Deployments()

看看informers.NewSharedInformerFactory()方法：

func NewSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration) SharedInformerFactory {
	return NewSharedInformerFactoryWithOptions(client, defaultResync)
}

func NewSharedInformerFactoryWithOptions(client kubernetes.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
	factory := &sharedInformerFactory{
		client:           client,
		namespace:        v1.NamespaceAll,
		defaultResync:    defaultResync,
		informers:        make(map[reflect.Type]cache.SharedIndexInformer),
		startedInformers: make(map[reflect.Type]bool),
		customResync:     make(map[reflect.Type]time.Duration),
	}

	// Apply all options
	for _, opt := range options {
		factory = opt(factory)
	}
	return factory
}

而sharedInformerFactory结构体实现了Apps()方法：

// vendor/k8s.io/client-go/informers/factory.go
func (f *sharedInformerFactory) Apps() apps.Interface {
	return apps.New(f, f.namespace, f.tweakListOptions)
}

// vendor/k8s.io/client-go/informers/apps/interface.go
func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
	return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
}

而group结构体实现了V1()方法：

func (g *group) V1() v1.Interface {
	return v1.New(g.factory, g.namespace, g.tweakListOptions)
}

func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
}

而version结构体实现了Deployments()方法:

func (v *version) Deployments() DeploymentInformer {
	return &deploymentInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
}

因此deploymentInformer 实例就生成了。

2. 初始化 sharedIndexInformer

func (f *deploymentInformer) Informer() cache.SharedIndexInformer {
	// vendor/k8s.io/client-go/informers/factory.go
	return f.factory.InformerFor(&appsv1.Deployment{}, f.defaultInformer)
}

func (f *deploymentInformer) defaultInformer(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
	return NewFilteredDeploymentInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
}

func NewFilteredDeploymentInformer(client kubernetes.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
	return cache.NewSharedIndexInformer(
		&cache.ListWatch{
			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
				if tweakListOptions != nil {
					tweakListOptions(&options)
				}
				return client.AppsV1().Deployments(namespace).List(context.TODO(), options)
			},
			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
				if tweakListOptions != nil {
					tweakListOptions(&options)
				}
				return client.AppsV1().Deployments(namespace).Watch(context.TODO(), options)
			},
		},
		&appsv1.Deployment{},
		resyncPeriod,
		indexers,
	)
}

这里有个ListWatch属性，informer会调用这里面的ListFunc和WatchFunc获取全量和增量的资源对象。

func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {
	f.lock.Lock()
	defer f.lock.Unlock()
	informerType := reflect.TypeOf(obj)
	// ...
	informer = newFunc(f.client, resyncPeriod) // 这里会生成informer
	f.informers[informerType] = informer

	return informer
}

生成后的sharedIndexInformer会绑定到shardInformerFactory的informers里

3. 初始化lister

lister := deploymentInformer.Lister()

// vendor/k8s.io/client-go/informers/apps/v1/deployment.go
func (f *deploymentInformer) Lister() v1.DeploymentLister {
	return v1.NewDeploymentLister(f.Informer().GetIndexer())
}

func NewDeploymentLister(indexer cache.Indexer) DeploymentLister {
	return &deploymentLister{indexer: indexer}
}

// f.Informer().GetIndexer()
func (f *deploymentInformer) defaultInformer(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
	return NewFilteredDeploymentInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
}

func (s *sharedIndexInformer) GetIndexer() Indexer {
	return s.indexer
}

可以看到lister最终是由cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}完成初始化的。

4. 启动informer

shardInformerFactory.Start(stopCh)

func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.shuttingDown {
		return
	}
	fmt.Println(1111, len(f.informers))
	for informerType, informer := range f.informers {
		if !f.startedInformers[informerType] {
			f.wg.Add(1)
			informer := informer
			go func() {
				defer f.wg.Done()
                informer.Run(stopCh) // Run()
			}()
			f.startedInformers[informerType] = true
		}
	}
}

主要是调用了informer.Run(stopCh)，那就再看下 sharedIndexInformer 的Run()方法

// vendor/k8s.io/client-go/tools/cache/shared_informer.go
func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) {
	// ...
	fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{
		KnownObjects:          s.indexer,
		EmitDeltaTypeReplaced: true,
	})

	cfg := &Config{
		Queue:            fifo,
		ListerWatcher:    s.listerWatcher,
		ObjectType:       s.objectType,
		FullResyncPeriod: s.resyncCheckPeriod,
		RetryOnError:     false,
		ShouldResync:     s.processor.shouldResync,

		Process:           s.HandleDeltas, // 定义process，处理fifo里的事件
		WatchErrorHandler: s.watchErrorHandler,
	}
    func() {
		s.startedLock.Lock()
		defer s.startedLock.Unlock()

		s.controller = New(cfg) // 初始化contrller
		s.controller.(*controller).clock = s.clock
		s.started = true
	}()

	// ...
	wg.StartWithChannel(processorStopCh, s.processor.run) // 运行processor
	// ...
	s.controller.Run(stopCh)
}

它做了以下几件事：

1、 初始化contrller，启动contrller；

2、启动processer

5. 启动contrller

func (c *controller) Run(stopCh <-chan struct{}) {
   // ...
   r := NewReflector(
      c.config.ListerWatcher,
      c.config.ObjectType,
      c.config.Queue,
      c.config.FullResyncPeriod,
   )
   // ...

   wg.StartWithChannel(stopCh, r.Run)

   wait.Until(c.processLoop, time.Second, stopCh)
   wg.Wait()
}

它做了以下几件事：

1、初始化Refector并启动；

2、执行processLoop

5.1. Refector的执行逻辑

func (r *Reflector) Run(stopCh <-chan struct{}) {
	klog.V(3).Infof("Starting reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name)
	wait.BackoffUntil(func() {
		if err := r.ListAndWatch(stopCh); err != nil { // 关键代码
			r.watchErrorHandler(r, err)
		}
	}, r.backoffManager, true, stopCh)
	klog.V(3).Infof("Stopping reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name)
}

ListAndWatch逻辑：

func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
	klog.V(3).Infof("Listing and watching %v from %s", r.expectedTypeName, r.name)

	err := r.list(stopCh) 
	if err != nil {
		return err
	}

	resyncerrc := make(chan error, 1)
	cancelCh := make(chan struct{})
	defer close(cancelCh)
	go func() {
		resyncCh, cleanup := r.resyncChan()
		defer func() {
			cleanup() // Call the last one written into cleanup
		}()
		// ...
	for {
		// ...
		start := r.clock.Now()
		w, err := r.listerWatcher.Watch(options)
		// ...

		err = watchHandler(start, w, r.store, r.expectedType, r.expectedGVK, r.name, r.expectedTypeName, r.setLastSyncResourceVersion, r.clock, resyncerrc, stopCh)
		retry.After(err)
		if err != nil {
			// ...
			return nil
		}
	}
}

先执行list，获取全量的资源，然后执行watch方法，进行增量添加。

func (r *Reflector) list(stopCh <-chan struct{}) error {
	// ...
	go func() {
		defer func() {
			// ...
		pager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
			return r.listerWatcher.List(opts)
		}))
		// ...

		list, paginatedResult, err = pager.List(context.Background(), options)
		// ...
		close(listCh)
	}()
	// ...
	items, err := meta.ExtractList(list)
	// ...
	if err := r.syncWith(items, resourceVersion); err != nil {
		return fmt.Errorf("unable to sync list result: %v", err)
	}
	// ...
	return nil
}
    
func (r *Reflector) syncWith(items []runtime.Object, resourceVersion string) error {
	found := make([]interface{}, 0, len(items))
	for _, item := range items {
		found = append(found, item)
	}
	return r.store.Replace(found, resourceVersion)
}

list的逻辑就是调用listerWatcher.List方法获取全量的资源，然后放入fifo。

watch的逻辑就是调用listerWatcher.Watch方法获取增量的资源，然后放入fifo。

5.2. processLoop执行逻辑

func (c *controller) processLoop() {
	for {
		obj, err := c.config.Queue.Pop(PopProcessFunc(c.config.Process))
		// ...
	}
}

它会不断从fifo队列里取出事件并处理，看看它的处理函数，它在初始化contrller时就定义了：

cfg := &Config{
        // ...
		Process:           s.HandleDeltas, // 定义process，处理fifo里的事件
		WatchErrorHandler: s.watchErrorHandler,
	}

func (s *sharedIndexInformer) HandleDeltas(obj interface{}) error {
	s.blockDeltas.Lock()
	defer s.blockDeltas.Unlock()
	if deltas, ok := obj.(Deltas); ok {
		return processDeltas(s, s.indexer, s.transform, deltas)
	}
	return errors.New("object given as Process argument is not Deltas")
}

关键就在这个processDeltas()函数：

// vendor/k8s.io/client-go/tools/cache/controller.go
func processDeltas(
	// Object which receives event notifications from the given deltas
	handler ResourceEventHandler,
	clientState Store,
	transformer TransformFunc,
	deltas Deltas,
) error {
	// from oldest to newest
	for _, d := range deltas {
		// ...
		switch d.Type {
		case Sync, Replaced, Added, Updated:
			if old, exists, err := clientState.Get(obj); err == nil && exists {
				if err := clientState.Update(obj); err != nil {
					return err
				}
				handler.OnUpdate(old, obj)
			} else {
				if err := clientState.Add(obj); err != nil {
					return err
				}
				handler.OnAdd(obj)
			}
		case Deleted:
            // 从缓存中删除事件
			if err := clientState.Delete(obj); err != nil {
				return err
			}
            // 将删除事件写入管道
			handler.OnDelete(obj)
		}
	}
	return nil
}

以删除事件为例，它做了两件事：

1、从缓存中删除这个事件；

2、将删除事件写入管道。

5.2.1. 从缓存中删除事件

// vendor/k8s.io/client-go/tools/cache/store.go
func (c *cache) Delete(obj interface{}) error {
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.cacheStorage.Delete(key)
	return nil
}

先调用keyFunc方法获取事件对应的key，然后删除这个key。

5.2.2. 将删除事件写入管道

handler实例对应的就是sharedIndexInformer实例，它实现了ResourceEventHandler接口

type ResourceEventHandler interface {
	OnAdd(obj interface{})
	OnUpdate(oldObj, newObj interface{})
	OnDelete(obj interface{})
}

那我们来看看sharedIndexInformer实现的OnDelete方法：

// vendor/k8s.io/client-go/tools/cache/shared_informer.go
func (s *sharedIndexInformer) OnAdd(obj interface{}) {
   s.cacheMutationDetector.AddObject(obj)
   s.processor.distribute(addNotification{newObj: obj}, false)
}

processor的distribute方法：

func (p *sharedProcessor) distribute(obj interface{}, sync bool) {
	p.listenersLock.RLock()
	defer p.listenersLock.RUnlock()

	for listener, isSyncing := range p.listeners {
		switch {
		case !sync:
			// non-sync messages are delivered to every listener
			listener.add(obj)
		case isSyncing:
			// sync messages are delivered to every syncing listener
			listener.add(obj)
		default:
			// skipping a sync obj for a non-syncing listener
		}
	}
}

再看看listener的add方法：

func (p *processorListener) add(notification interface{}) {
	p.addCh <- notification
}

逻辑很简单，就是把事件写入addCh管道。有写入肯定就有读取，下面看看processor的处理逻辑。

6. 启动processor

func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) {
	// ...
	wg.StartWithChannel(processorStopCh, s.processor.run) // 运行processor
	// ...
	s.controller.Run(stopCh)
}

和contrller一样，processor也是在sharedIndexInformer 里启动的。

func (p *sharedProcessor) run(stopCh <-chan struct{}) {
	func() {
		p.listenersLock.RLock()
		defer p.listenersLock.RUnlock()
		for listener := range p.listeners {
			p.wg.Start(listener.run)  // 调用processorListener的run()
            p.wg.Start(listener.pop)  // 调用调用processorListener的run的pop()
		}
		p.listenersStarted = true
	}()
	// ...
}

它实际上使用两个管道实现了类型队列的功能：

1、pop方法里先从addCh里读取数据，然后放入nextCh；

2、run方法里从nextCh读取事件并回调用户定义的回调函数。

这里我们只看run方法。

6.1. run的执行逻辑

func (p *processorListener) run() {
	stopCh := make(chan struct{})
	wait.Until(func() {
		for next := range p.nextCh {
			switch notification := next.(type) {
			case updateNotification:
				p.handler.OnUpdate(notification.oldObj, notification.newObj)
			case addNotification:
				p.handler.OnAdd(notification.newObj)
			case deleteNotification:
				p.handler.OnDelete(notification.oldObj)
			default:
				utilruntime.HandleError(fmt.Errorf("unrecognized notification: %T", next))
			}
		}
		// the only way to get here is if the p.nextCh is empty and closed
		close(stopCh)
	}, 1*time.Second, stopCh)
}

p.handler就是用户自定义的handler：

sharedIndexInformer.AddEventHandler(
		cache.ResourceEventHandlerFuncs{
			AddFunc:    onAdd,
			UpdateFunc: onUpdate,
			DeleteFunc: onDelete,
		})

AddEventHandler方法：

// vendor/k8s.io/client-go/tools/cache/shared_informer.go
func (s *sharedIndexInformer) AddEventHandler(handler ResourceEventHandler) (ResourceEventHandlerRegistration, error) {
	return s.AddEventHandlerWithResyncPeriod(handler, s.defaultEventHandlerResyncPeriod)
}

AddEventHandlerWithResyncPeriod方法：

func (s *sharedIndexInformer) AddEventHandlerWithResyncPeriod(handler ResourceEventHandler, resyncPeriod time.Duration) (ResourceEventHandlerRegistration, error) {
	// ...
    //生成processListener实例
	listener := newProcessListener(handler, resyncPeriod, determineResyncPeriod(resyncPeriod, s.resyncCheckPeriod), s.clock.Now(), initialBufferSize)
    // ...
	handle := s.processor.addListener(listener)
	for _, item := range s.indexer.List() {
		listener.add(addNotification{newObj: item})
	}
	return handle, nil
}

可以看到processListener实例会绑定用户自定义的handler，再看看ResourceEventHandlerFuncs这个结构体：

type ResourceEventHandlerFuncs struct {
	AddFunc    func(obj interface{})
	UpdateFunc func(oldObj, newObj interface{})
	DeleteFunc func(obj interface{})
}

// OnDelete方法
func (r ResourceEventHandlerFuncs) OnDelete(obj interface{}) {
	if r.DeleteFunc != nil {
		r.DeleteFunc(obj) // 回调
	}
}

最终回调用户自定义的处理函数，整个流程就介绍完毕了。

7. indexer

informer的本地缓存，存储全量最新的资源信息，因此客户端不用直接和apiserver交互去获取资源信息，减轻apiserver的压力。

我专门把它的功能抽出来做了一个示例：

package main

import (
	"fmt"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"
)

func main() {
	indexers := cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}
	indexer := cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, indexers)
	pod1 := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pod-1",
			Namespace: "default",
		},
		Spec: v1.PodSpec{
			NodeName: "node1",
		},
	}
	pod2 := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pod-2",
			Namespace: "default",
		},
		Spec: v1.PodSpec{
			NodeName: "node2",
		},
	}
	pod3 := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pod-3",
			Namespace: "kube-system",
		},
		Spec: v1.PodSpec{
			NodeName: "node2",
		},
	}
	indexer.Add(pod1)
	indexer.Add(pod2)
	indexer.Add(pod3)
	fmt.Println(len(indexer.List()))
	pods, _ := indexer.ByIndex("namespace", "default")
	for _, v := range pods {
		fmt.Println(v.(*v1.Pod).Name)
	}
}