From fdb22419dd326ea08a92fc0944543dc5a37a1ae3 Mon Sep 17 00:00:00 2001 From: Oren Shomron Date: Tue, 28 Apr 2020 16:47:37 -0400 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9C=A8=20Allow=20removing=20individual?= =?UTF-8?q?=20informers=20from=20the=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds support for removing individual informers from the cache using the new Remove() method. This is allowed before or after the cache has been started. Informers are stopped at the time of removal - once stopped, they will no longer deliver events to registered event handlers, and registered watched will be aborted. Also adds non-blocking API for getting informers without waiting for their cache to sync - GetInformerNonBlocking(). Signed-off-by: Oren Shomron --- pkg/cache/cache.go | 2 +- pkg/cache/informer_cache.go | 30 +++++++++ pkg/cache/internal/deleg_map.go | 14 ++++ pkg/cache/internal/informers_map.go | 42 ++++++++++-- pkg/cache/internal/internal_suite_test.go | 14 ++++ pkg/cache/internal/sync.go | 79 +++++++++++++++++++++++ pkg/cache/internal/sync_test.go | 71 ++++++++++++++++++++ 7 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 pkg/cache/internal/internal_suite_test.go create mode 100644 pkg/cache/internal/sync.go create mode 100644 pkg/cache/internal/sync_test.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 71dfbd0454..a5f2cec704 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -126,7 +126,7 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { // Construct a new Mapper if unset if opts.Mapper == nil { var err error - opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config) + opts.Mapper, err = apiutil.NewDynamicRESTMapper(config) if err != nil { log.WithName("setup").Error(err, "Failed to get API Group-Resources") return opts, fmt.Errorf("could not create RESTMapper from config") diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 8ec3b921d9..e2ea65608e 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -22,6 +22,7 @@ import ( "reflect" "strings" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -159,6 +160,24 @@ func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (In return i.Informer, err } +// GetInformerNonBlocking returns the informer for the obj without waiting for its cache to sync. +func (ip *informerCache) GetInformerNonBlocking(obj runtime.Object) (Informer, error) { + gvk, err := apiutil.GVKForObject(obj, ip.Scheme) + if err != nil { + return nil, err + } + + // Use a cancelled context to signal non-blocking + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, i, err := ip.InformersMap.Get(ctx, gvk, obj) + if err != nil && !apierrors.IsTimeout(err) { + return nil, err + } + return i.Informer, nil +} + // NeedLeaderElection implements the LeaderElectionRunnable interface // to indicate that this can be started without requiring the leader lock func (ip *informerCache) NeedLeaderElection() bool { @@ -216,3 +235,14 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) return indexer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) } + +// Remove removes an informer specified by the obj argument from the cache and stops it if it existed. +func (ip *informerCache) Remove(obj runtime.Object) error { + gvk, err := apiutil.GVKForObject(obj, ip.Scheme) + if err != nil { + return err + } + + ip.InformersMap.Remove(gvk, obj) + return nil +} diff --git a/pkg/cache/internal/deleg_map.go b/pkg/cache/internal/deleg_map.go index a7ee643482..53e9a7c231 100644 --- a/pkg/cache/internal/deleg_map.go +++ b/pkg/cache/internal/deleg_map.go @@ -92,6 +92,20 @@ func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj return m.structured.Get(ctx, gvk, obj) } +// Remove will remove an new Informer from the InformersMap and stop it if it exists. +func (m *InformersMap) Remove(gvk schema.GroupVersionKind, obj runtime.Object) { + _, isUnstructured := obj.(*unstructured.Unstructured) + _, isUnstructuredList := obj.(*unstructured.UnstructuredList) + isUnstructured = isUnstructured || isUnstructuredList + + switch { + case isUnstructured: + m.unstructured.Remove(gvk) + default: + m.structured.Remove(gvk) + } +} + // newStructuredInformersMap creates a new InformersMap for structured objects. func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap { return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createStructuredListWatch) diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go index bc450d8781..c3b21f7942 100644 --- a/pkg/cache/internal/informers_map.go +++ b/pkg/cache/internal/informers_map.go @@ -70,6 +70,9 @@ type MapEntry struct { // CacheReader wraps Informer and implements the CacheReader interface for a single type Reader CacheReader + + // Stop can be used to stop this individual informer without stopping the entire specificInformersMap. + stop chan struct{} } // specificInformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. @@ -121,6 +124,17 @@ type specificInformersMap struct { namespace string } +// Start starts the informer managed by a MapEntry. +// Blocks until the informer stops. The informer can be stopped +// either individually (via the entry's stop channel) or globally +// via the provided stop argument. +func (e *MapEntry) Start(stop <-chan struct{}) { + // Stop on either the whole map stopping or just this informer being removed. + internalStop, cancel := anyOf(stop, e.stop) + defer cancel() + e.Informer.Run(internalStop) +} + // Start calls Run on each of the informers and sets started to true. Blocks on the context. // It doesn't return start because it can't return an error, and it's not a runnable directly. func (ip *specificInformersMap) Start(ctx context.Context) { @@ -132,8 +146,8 @@ func (ip *specificInformersMap) Start(ctx context.Context) { ip.stop = ctx.Done() // Start each informer - for _, informer := range ip.informersByGVK { - go informer.Informer.Run(ctx.Done()) + for _, entry := range ip.informersByGVK { + go entry.Start(ctx.Done()) } // Set started to true so we immediately start any informers added later. @@ -183,8 +197,12 @@ func (ip *specificInformersMap) Get(ctx context.Context, gvk schema.GroupVersion if started && !i.Informer.HasSynced() { // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. - if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { - return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) + // Cancel for context, informer stopping, or entire map stopping. + syncStop, cancel := mergeChan(ctx.Done(), i.stop, ip.stop) + defer cancel() + if !cache.WaitForCacheSync(syncStop, i.Informer.HasSynced) { + // Return entry even on timeout - caller may have intended a non-blocking fetch. + return started, i, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) } } @@ -214,6 +232,7 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob i := &MapEntry{ Informer: ni, Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk}, + stop: make(chan struct{}), } ip.informersByGVK[gvk] = i @@ -221,11 +240,24 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob // TODO(seans): write thorough tests and document what happens here - can you add indexers? // can you add eventhandlers? if ip.started { - go i.Informer.Run(ip.stop) + go i.Start(ip.stop) } return i, ip.started, nil } +// Remove removes an informer entry and stops it if it was running. +func (ip *specificInformersMap) Remove(gvk schema.GroupVersionKind) { + ip.mu.Lock() + defer ip.mu.Unlock() + + entry, ok := ip.informersByGVK[gvk] + if !ok { + return + } + close(entry.stop) + delete(ip.informersByGVK, gvk) +} + // newListWatch returns a new ListWatch object that can be used to create a SharedIndexInformer. func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the diff --git a/pkg/cache/internal/internal_suite_test.go b/pkg/cache/internal/internal_suite_test.go new file mode 100644 index 0000000000..6686d14eba --- /dev/null +++ b/pkg/cache/internal/internal_suite_test.go @@ -0,0 +1,14 @@ +package internal + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" +) + +func TestCacheInternal(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "Cache Internal Suite", []Reporter{printer.NewlineReporter{}}) +} diff --git a/pkg/cache/internal/sync.go b/pkg/cache/internal/sync.go new file mode 100644 index 0000000000..94102edfd3 --- /dev/null +++ b/pkg/cache/internal/sync.go @@ -0,0 +1,79 @@ +package internal + +import ( + "context" + "sync" +) + +// anyOf returns a "done" channel that is closed when any of its input channels +// are closed or when the retuned cancel function is called, whichever comes first. +// +// The cancel function should always be called by the caller to ensure +// resources are properly released. +func anyOf(ch ...<-chan struct{}) (<-chan struct{}, context.CancelFunc) { + var once sync.Once + cancel := make(chan struct{}) + cancelFunc := func() { + once.Do(func() { + close(cancel) + }) + } + return anyInternal(append(ch, cancel)...), cancelFunc +} + +func anyInternal(ch ...<-chan struct{}) <-chan struct{} { + switch len(ch) { + case 0: + return nil + case 1: + return ch[0] + } + + done := make(chan struct{}) + go func() { + defer close(done) + + switch len(ch) { + case 2: + // This case saves a recursion + goroutine when there are exactly 2 channels. + select { + case <-ch[0]: + case <-ch[1]: + } + default: + // >=3 channels to merge + select { + case <-ch[0]: + case <-ch[1]: + case <-ch[2]: + case <-anyInternal(append(ch[3:], done)...): + } + } + }() + + return done +} + +// mergeChan returns a channel that is closed when any of the input channels are signaled. +// The caller must call the returned CancelFunc to ensure no resources are leaked. +func mergeChan(a, b, c <-chan struct{}) (<-chan struct{}, context.CancelFunc) { + var once sync.Once + out := make(chan struct{}) + cancel := make(chan struct{}) + cancelFunc := func() { + once.Do(func() { + close(cancel) + }) + } + go func() { + defer close(out) + select { + case <-a: + case <-b: + case <-c: + case <-cancel: + } + }() + + return out, cancelFunc +} diff --git a/pkg/cache/internal/sync_test.go b/pkg/cache/internal/sync_test.go new file mode 100644 index 0000000000..21861c44d8 --- /dev/null +++ b/pkg/cache/internal/sync_test.go @@ -0,0 +1,71 @@ +package internal + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("anyOf", func() { + // Generate contexts for different number of input channels + for n := 0; n < 4; n++ { + n := n + Context(fmt.Sprintf("with %d channels", n), func() { + var ( + channels []chan struct{} + done <-chan struct{} + cancel context.CancelFunc + ) + BeforeEach(func() { + channels = make([]chan struct{}, n) + in := make([]<-chan struct{}, n) + for i := 0; i < n; i++ { + ch := make(chan struct{}) + channels[i] = ch + in[i] = ch + } + done, cancel = anyOf(in...) + }) + AfterEach(func() { + cancel() + }) + + It("isn't closed initially", func() { + select { + case <-done: + Fail("done was closed before cancel") + case <-time.After(5 * time.Millisecond): + // Ok. + } + }) + + // Verify that done is closed when we call cancel explicitly. + It("closes when cancelled", func() { + cancel() + select { + case <-done: + // Ok. + case <-time.After(5 * time.Millisecond): + Fail("timed out waiting for cancel") + } + }) + + // Generate test cases for closing each individual channel. + // Verify that done is closed in response. + for i := 0; i < n; i++ { + i := i + It(fmt.Sprintf("closes when channel %d is closed", i), func() { + close(channels[i]) + select { + case <-done: + // Ok. + case <-time.After(5 * time.Millisecond): + Fail("timed out waiting for cancel") + } + }) + } + }) + } +}) From 579fb28d0a2dd7c5946e9a13844634edef0cf3fe Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Wed, 30 Sep 2020 23:02:04 +0000 Subject: [PATCH 2/6] Enable starting/stopping controllers and their caches --- pkg/cache/cache.go | 3 +++ pkg/cache/informer_cache.go | 2 +- pkg/cache/informertest/fake_cache.go | 5 +++++ pkg/cache/multi_namespace_cache.go | 9 +++++++++ pkg/internal/controller/controller.go | 23 +++++++++++++++++++++-- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index a5f2cec704..7cb8cdccac 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -58,6 +58,9 @@ type Informers interface { // of the underlying object. GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) + // Remove the informer for the given object. + Remove(ctx context.Context, obj runtime.Object) error + // Start runs all the informers known to this cache until the context is closed. // It blocks. Start(ctx context.Context) error diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index e2ea65608e..6d4807c074 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -237,7 +237,7 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) } // Remove removes an informer specified by the obj argument from the cache and stops it if it existed. -func (ip *informerCache) Remove(obj runtime.Object) error { +func (ip *informerCache) Remove(ctx context.Context, obj runtime.Object) error { gvk, err := apiutil.GVKForObject(obj, ip.Scheme) if err != nil { return err diff --git a/pkg/cache/informertest/fake_cache.go b/pkg/cache/informertest/fake_cache.go index eb78e0bb65..f2548fb7c8 100644 --- a/pkg/cache/informertest/fake_cache.go +++ b/pkg/cache/informertest/fake_cache.go @@ -125,6 +125,11 @@ func (c *FakeInformers) Start(ctx context.Context) error { return c.Error } +// Remove implements Cache +func (c *FakeInformers) Remove(ctx context.Context, obj runtime.Object) error { + return c.Error +} + // IndexField implements Cache func (c *FakeInformers) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { return nil diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index f0e18c09b0..3dfd4aaf60 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -94,6 +94,15 @@ func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema return &multiNamespaceInformer{namespaceToInformer: informers}, nil } +func (c *multiNamespaceCache) Remove(ctx context.Context, obj runtime.Object) error { + for _, cache := range c.namespaceToCache { + if err := cache.Remove(ctx, obj); err != nil { + return err + } + } + return nil +} + func (c *multiNamespaceCache) Start(ctx context.Context) error { for ns, cache := range c.namespaceToCache { go func(ns string, cache Cache) { diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index 83866606c2..cc1bd1d9f1 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -79,6 +79,9 @@ type Controller struct { // undergo a major refactoring and redesign to allow for context to not be stored in a struct. ctx context.Context + // saveWatches indicates not to clear startWatches when the controller is stopped. + saveWatches bool + // startWatches maintains a list of sources, handlers, and predicates to start when the controller is started. startWatches []watchDescription @@ -180,9 +183,15 @@ func (c *Controller) Start(ctx context.Context) error { // All the watches have been started, we can reset the local slice. // - // We should never hold watches more than necessary, each watch source can hold a backing cache, + // We should not usually hold watches more than necessary, each watch source can hold a backing cache, // which won't be garbage collected if we hold a reference to it. - c.startWatches = nil + + // The exception to this is when the saveWatches is set to true, + // meaning the controller is intending to run Start() again. + // In this case it needs knowledge of the watches for when the controller is restarted. + if !c.saveWatches { + c.startWatches = nil + } if c.JitterPeriod == 0 { c.JitterPeriod = 1 * time.Second @@ -302,3 +311,13 @@ func (c *Controller) InjectFunc(f inject.Func) error { func (c *Controller) updateMetrics(reconcileTime time.Duration) { ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds()) } + +// ResetStart sets Started to false to enable running Start on the controller again. +func (c *Controller) ResetStart() { + c.Started = false +} + +// SaveWatches indicates that watches should not be cleared when the controller is stopped. +func (c *Controller) SaveWatches() { + c.saveWatches = true +} From 1cc6939a171dd2c8fa3ea2ab2e00ac419b810bbd Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Mon, 19 Oct 2020 21:19:52 +0000 Subject: [PATCH 3/6] Add EventHandler reference counting --- pkg/cache/cache.go | 8 +++ pkg/cache/informer_cache.go | 3 +- pkg/cache/internal/counting_informer.go | 86 +++++++++++++++++++++++++ pkg/cache/internal/deleg_map.go | 6 +- pkg/cache/internal/informers_map.go | 23 +++++-- pkg/cache/multi_namespace_cache.go | 17 +++++ pkg/internal/controller/controller.go | 14 +++- pkg/source/source.go | 23 +++++++ 8 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 pkg/cache/internal/counting_informer.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 7cb8cdccac..f1557885bc 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -87,6 +87,14 @@ type Informer interface { AddIndexers(indexers toolscache.Indexers) error //HasSynced return true if the informers underlying store has synced HasSynced() bool + + // RemoveEventHandler currently just decrements a the count of event handlers + // The goals it to have SharedInformer support RemoveEventHandler (and actually remove + // the handler instead of just decrementing a count). + RemoveEventHandler(id int) error + + // CountEventHandlers returns the number of event handlers added to an informer. + CountEventHandlers() int } // Options are the optional arguments for creating a new InformersMap object diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 6d4807c074..17b8ab4e18 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -243,6 +243,5 @@ func (ip *informerCache) Remove(ctx context.Context, obj runtime.Object) error { return err } - ip.InformersMap.Remove(gvk, obj) - return nil + return ip.InformersMap.Remove(gvk, obj) } diff --git a/pkg/cache/internal/counting_informer.go b/pkg/cache/internal/counting_informer.go new file mode 100644 index 0000000000..2567b00a83 --- /dev/null +++ b/pkg/cache/internal/counting_informer.go @@ -0,0 +1,86 @@ +package internal + +import ( + "fmt" + "time" + + "k8s.io/client-go/tools/cache" +) + +// CountingInformer exposes a way to track the number of EventHandlers +// registered on an Informer. +type CountingInformer interface { + cache.SharedIndexInformer + CountEventHandlers() int + RemoveEventHandler(id int) error +} + +// HandlerCountingInformer implements the CountingInformer. +// It increments the count every time AddEventHandler is called, +// and decrements the count every time RemoveEventHandler is called. +// +// It doesn't actually RemoveEventHandlers because that feature is not implemented +// in client-go, but we're are naming it this way to suggest what the interface would look +// like if/when it does get added to client-go. +// +// We can get rid of this if apimachinery adds the ability to retrieve this from the SharedIndexInformer +// but until then, we have to track it ourselves +type HandlerCountingInformer struct { + // Informer is the cached informer + informer cache.SharedIndexInformer + + // count indicates the number of EventHandlers registered on the informer + count int +} + +func (i *HandlerCountingInformer) RemoveEventHandler(id int) error { + i.count-- + fmt.Printf("decrement, count is %+v\n", i.count) + return nil +} + +func (i *HandlerCountingInformer) AddEventHandler(handler cache.ResourceEventHandler) { + i.count++ + fmt.Printf("increment, count is %+v\n", i.count) + i.informer.AddEventHandler(handler) +} + +func (i *HandlerCountingInformer) CountEventHandlers() int { + return i.count +} + +func (i *HandlerCountingInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) { + i.count++ + i.informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) +} +func (i *HandlerCountingInformer) AddIndexers(indexers cache.Indexers) error { + return i.informer.AddIndexers(indexers) +} + +func (i *HandlerCountingInformer) HasSynced() bool { + return i.informer.HasSynced() +} + +func (i *HandlerCountingInformer) GetStore() cache.Store { + return i.informer.GetStore() +} + +func (i *HandlerCountingInformer) GetController() cache.Controller { + return i.informer.GetController() +} + +func (i *HandlerCountingInformer) LastSyncResourceVersion() string { + return i.informer.LastSyncResourceVersion() +} + +func (i *HandlerCountingInformer) SetWatchErrorHandler(handler cache.WatchErrorHandler) error { + return i.informer.SetWatchErrorHandler(handler) +} + +func (i *HandlerCountingInformer) GetIndexer() cache.Indexer { + return i.informer.GetIndexer() +} + +func (i *HandlerCountingInformer) Run(stopCh <-chan struct{}) { + i.informer.Run(stopCh) +} diff --git a/pkg/cache/internal/deleg_map.go b/pkg/cache/internal/deleg_map.go index 53e9a7c231..4489b37aaa 100644 --- a/pkg/cache/internal/deleg_map.go +++ b/pkg/cache/internal/deleg_map.go @@ -93,16 +93,16 @@ func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj } // Remove will remove an new Informer from the InformersMap and stop it if it exists. -func (m *InformersMap) Remove(gvk schema.GroupVersionKind, obj runtime.Object) { +func (m *InformersMap) Remove(gvk schema.GroupVersionKind, obj runtime.Object) error { _, isUnstructured := obj.(*unstructured.Unstructured) _, isUnstructuredList := obj.(*unstructured.UnstructuredList) isUnstructured = isUnstructured || isUnstructuredList switch { case isUnstructured: - m.unstructured.Remove(gvk) + return m.unstructured.Remove(gvk) default: - m.structured.Remove(gvk) + return m.structured.Remove(gvk) } } diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go index c3b21f7942..5669b03ab3 100644 --- a/pkg/cache/internal/informers_map.go +++ b/pkg/cache/internal/informers_map.go @@ -65,8 +65,8 @@ func newSpecificInformersMap(config *rest.Config, // MapEntry contains the cached data for an Informer type MapEntry struct { - // Informer is the cached informer - Informer cache.SharedIndexInformer + // Informer is a SharedIndexInformer with addition count and remove event handler functionality. + Informer CountingInformer // CacheReader wraps Informer and implements the CacheReader interface for a single type Reader CacheReader @@ -230,7 +230,7 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, }) i := &MapEntry{ - Informer: ni, + Informer: &HandlerCountingInformer{ni, 0}, Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk}, stop: make(chan struct{}), } @@ -241,21 +241,34 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob // can you add eventhandlers? if ip.started { go i.Start(ip.stop) + //go i.Start(StopOptions{ + // StopChannel: ip.stop, + //}) } return i, ip.started, nil } // Remove removes an informer entry and stops it if it was running. -func (ip *specificInformersMap) Remove(gvk schema.GroupVersionKind) { +func (ip *specificInformersMap) Remove(gvk schema.GroupVersionKind) error { ip.mu.Lock() defer ip.mu.Unlock() entry, ok := ip.informersByGVK[gvk] if !ok { - return + return nil } + + chInformer, ok := entry.Informer.(*HandlerCountingInformer) + if !ok { + return fmt.Errorf("entry informer is not a HandlerCountingInformer") + } + if chInformer.CountEventHandlers() != 0 { + return fmt.Errorf("attempting to remove informer with %d references", chInformer.CountEventHandlers()) + } + close(entry.stop) delete(ip.informersByGVK, gvk) + return nil } // newListWatch returns a new ListWatch object that can be used to create a SharedIndexInformer. diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index 3dfd4aaf60..0ab61623e7 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -195,6 +195,23 @@ type multiNamespaceInformer struct { var _ Informer = &multiNamespaceInformer{} +func (i *multiNamespaceInformer) CountEventHandlers() int { + total := 0 + for _, informer := range i.namespaceToInformer { + total += informer.CountEventHandlers() + } + return total +} + +func (i *multiNamespaceInformer) RemoveEventHandler(id int) error { + for _, informer := range i.namespaceToInformer { + if err := informer.RemoveEventHandler(id); err != nil { + return err + } + } + return nil +} + // AddEventHandler adds the handler to each namespaced informer func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) { for _, informer := range i.namespaceToInformer { diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index cc1bd1d9f1..3cbd853e1d 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -158,9 +158,17 @@ func (c *Controller) Start(ctx context.Context) error { // caches to sync so that they have a chance to register their intendeded // caches. for _, watch := range c.startWatches { - c.Log.Info("Starting EventSource", "source", watch.src) - if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil { - return err + stoppableSource, ok := watch.src.(source.StoppableSource) + if ok { + // TODO: use errgroup or waitgroup to not return until all goros have exited + // (or something else to prevent leaks) + go stoppableSource.StartStoppable(ctx, watch.handler, c.Queue, watch.predicates...) + c.Log.Info("Starting STOPPABLE EventSource", "source", watch.src) + } else { + c.Log.Info("Starting EventSource", "source", watch.src) + if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil { + return err + } } } diff --git a/pkg/source/source.go b/pkg/source/source.go index fe0e47150a..a6711ef66f 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -64,6 +64,14 @@ type SyncingSource interface { WaitForSync(ctx context.Context) error } +// StoppableSource expands the Source interface to add a start method that +// blocks on the context's Done channel, so that we know when the controller has +// been stopped and can remove/decremnt the EventHandler count on the informer appropriately. +type StoppableSource interface { + Source + StartStoppable(context.Context, handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error +} + // NewKindWithCache creates a Source without InjectCache, so that it is assured that the given cache is used // and not overwritten. It can be used to watch objects in a different cluster by passing the cache // from that other cluster @@ -123,6 +131,21 @@ func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue w return nil } +// StartStoppable blocks for start to finish and then calls RemoveEventHandler on the kind's informer. +func (ks *Kind) StartStoppable(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, + prct ...predicate.Predicate) error { + i, err := ks.cache.GetInformer(ctx, ks.Type) + if err != nil { + return err + } + if err := ks.Start(ctx, handler, queue, prct...); err != nil { + return err + } + <-ctx.Done() + i.RemoveEventHandler(-1) + return nil +} + func (ks *Kind) String() string { if ks.Type != nil && ks.Type.GetObjectKind() != nil { return fmt.Sprintf("kind source: %v", ks.Type.GetObjectKind().GroupVersionKind().String()) From 30182282dab07467c1b208f5c20a1fdd782d5740 Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Mon, 19 Oct 2020 21:34:30 +0000 Subject: [PATCH 4/6] Stub out RunWithStopOptions --- pkg/cache/cache.go | 17 +++++++++++++++++ pkg/cache/internal/counting_informer.go | 19 +++++++++++++++++++ pkg/cache/internal/informers_map.go | 21 ++++++++++++++++++++- pkg/cache/multi_namespace_cache.go | 9 +++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index f1557885bc..fc95933065 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -95,8 +95,25 @@ type Informer interface { // CountEventHandlers returns the number of event handlers added to an informer. CountEventHandlers() int + + // RunWithStopOptions runs the informer and provides options to be checked that + // would indicate under what conditions the informer should stop. + RunWithStopOptions(stopOptions internal.StopOptions) internal.StopReason } +// StopOptions let the caller specify which conditions to stop the informer. +type StopOptions struct { + // StopChannel stops the Informer when it receives a close signal. + StopChannel <-chan struct{} + + // OnListError inspects errors returned from the infromer's underlying refloector, + // and based on the error determines whether or not to stop the informer. + OnListError func(error) bool +} + +// StopReason is a custom typed error that indicates how the informer was stopped. +type StopReason error + // Options are the optional arguments for creating a new InformersMap object type Options struct { // Scheme is the scheme to use for mapping objects to GroupVersionKinds diff --git a/pkg/cache/internal/counting_informer.go b/pkg/cache/internal/counting_informer.go index 2567b00a83..297f55adc0 100644 --- a/pkg/cache/internal/counting_informer.go +++ b/pkg/cache/internal/counting_informer.go @@ -13,6 +13,7 @@ type CountingInformer interface { cache.SharedIndexInformer CountEventHandlers() int RemoveEventHandler(id int) error + RunWithStopOptions(stopOptions StopOptions) StopReason } // HandlerCountingInformer implements the CountingInformer. @@ -84,3 +85,21 @@ func (i *HandlerCountingInformer) GetIndexer() cache.Indexer { func (i *HandlerCountingInformer) Run(stopCh <-chan struct{}) { i.informer.Run(stopCh) } + +func (i *HandlerCountingInformer) RunWithStopOptions(stopOptions StopOptions) StopReason { + i.Run(stopOptions.StopChannel) + return nil +} + +// StopOptions let the caller specify which conditions to stop the informer. +type StopOptions struct { + // StopChannel stops the Informer when it receives a close signal. + StopChannel <-chan struct{} + + // OnListError inspects errors returned from the infromer's underlying refloector, + // and based on the error determines whether or not to stop the informer. + OnListError func(error) bool +} + +// StopReason is a custom typed error that indicates how the informer was stopped. +type StopReason error diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go index 5669b03ab3..0daaf23f02 100644 --- a/pkg/cache/internal/informers_map.go +++ b/pkg/cache/internal/informers_map.go @@ -135,8 +135,17 @@ func (e *MapEntry) Start(stop <-chan struct{}) { e.Informer.Run(internalStop) } +func (e *MapEntry) StartWithStopOptions(stopOptions StopOptions) { + // Stop on either the whole map stopping or just this informer being removed. + internalStop, cancel := anyOf(stopOptions.StopChannel, e.stop) + stopOptions.StopChannel = internalStop + defer cancel() + e.Informer.RunWithStopOptions(stopOptions) +} + // Start calls Run on each of the informers and sets started to true. Blocks on the context. // It doesn't return start because it can't return an error, and it's not a runnable directly. +// TODO: take in start options, bubble up to builder func (ip *specificInformersMap) Start(ctx context.Context) { func() { ip.mu.Lock() @@ -147,7 +156,10 @@ func (ip *specificInformersMap) Start(ctx context.Context) { // Start each informer for _, entry := range ip.informersByGVK { - go entry.Start(ctx.Done()) + //go entry.Start(ctx.Done()) + go entry.StartWithStopOptions(StopOptions{ + StopChannel: ctx.Done(), + }) } // Set started to true so we immediately start any informers added later. @@ -157,6 +169,13 @@ func (ip *specificInformersMap) Start(ctx context.Context) { <-ctx.Done() } +// StartWithStopOptions exposes a way to start an informer with user defined stopOptions +// We would plumb stopOptions from the builder (where the user would define them), down here through to the +// informer. +func (ip *specificInformersMap) StartWithStopOptions(ctx context.Context, stopOptions StopOptions) { + // TODO: implement once SharedIndexInformer in client-go supports RunWithStopOptions +} + func (ip *specificInformersMap) waitForStarted(ctx context.Context) bool { select { case <-ip.startWait: diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index 0ab61623e7..392a3a7777 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -195,6 +196,14 @@ type multiNamespaceInformer struct { var _ Informer = &multiNamespaceInformer{} +func (i *multiNamespaceInformer) RunWithStopOptions(stopOptions internal.StopOptions) internal.StopReason { + // TODO: don't leak goro - collect them with a errgroup or waitgroup + for _, informer := range i.namespaceToInformer { + go informer.RunWithStopOptions(stopOptions) + } + return nil +} + func (i *multiNamespaceInformer) CountEventHandlers() int { total := 0 for _, informer := range i.namespaceToInformer { From 55489254a15a6a3cb51495a7bf639d6773a6238b Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Wed, 30 Sep 2020 23:04:13 +0000 Subject: [PATCH 5/6] Create ConditionalController helper to enable wait and watch --- go.mod | 11 +- go.sum | 67 +++++-- pkg/builder/controller.go | 43 +++- pkg/builder/options.go | 23 +++ pkg/controller/conditional_controller.go | 173 ++++++++++++++++ pkg/controller/conditional_controller_test.go | 184 ++++++++++++++++++ pkg/controller/testdata/crds/foocrd.yaml | 17 ++ pkg/controller/testdata/foo/foo_types.go | 36 ++++ 8 files changed, 533 insertions(+), 21 deletions(-) create mode 100644 pkg/controller/conditional_controller.go create mode 100644 pkg/controller/conditional_controller_test.go create mode 100644 pkg/controller/testdata/crds/foocrd.yaml create mode 100644 pkg/controller/testdata/foo/foo_types.go diff --git a/go.mod b/go.mod index 3cf8801920..26996aaf9d 100644 --- a/go.mod +++ b/go.mod @@ -3,27 +3,36 @@ module sigs.k8s.io/controller-runtime go 1.15 require ( + github.com/coreos/go-etcd v2.0.0+incompatible // indirect + github.com/cpuguy83/go-md2man v1.0.10 // indirect + github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 // indirect github.com/evanphx/json-patch v4.9.0+incompatible github.com/fsnotify/fsnotify v1.4.9 github.com/go-logr/logr v0.2.1 github.com/go-logr/zapr v0.2.0 github.com/google/go-cmp v0.5.2 // indirect github.com/googleapis/gnostic v0.5.1 // indirect + github.com/gophercloud/gophercloud v0.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.10 // indirect github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0 + github.com/prometheus/common v0.10.0 + github.com/robfig/cron v1.2.0 + github.com/spf13/pflag v1.0.5 + github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 // indirect go.uber.org/goleak v1.1.10 go.uber.org/zap v1.15.0 + golang.org/x/text v0.3.3 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gomodules.xyz/jsonpatch/v2 v2.1.0 google.golang.org/appengine v1.6.6 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect k8s.io/api v0.19.2 k8s.io/apiextensions-apiserver v0.19.2 k8s.io/apimachinery v0.19.2 k8s.io/client-go v0.19.2 - k8s.io/utils v0.0.0-20200912215256-4140de9c8800 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 418b319378..223d3a68e8 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -62,13 +64,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -76,6 +81,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -87,6 +93,7 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= @@ -167,6 +174,7 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -201,10 +209,13 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -227,6 +238,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -319,16 +331,20 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -336,6 +352,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -343,6 +360,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -350,14 +368,13 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -366,6 +383,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -374,24 +392,22 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -400,6 +416,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -416,7 +433,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -424,8 +440,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -445,6 +461,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -462,11 +479,14 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -480,6 +500,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -491,6 +512,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -499,7 +521,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -524,6 +545,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -531,12 +553,12 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -548,9 +570,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -567,6 +587,7 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -579,10 +600,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -610,32 +631,50 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= +k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/builder/controller.go b/pkg/builder/controller.go index 047fc6d410..3640f6f2c4 100644 --- a/pkg/builder/controller.go +++ b/pkg/builder/controller.go @@ -19,9 +19,11 @@ package builder import ( "fmt" "strings" + "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -57,9 +59,11 @@ func ControllerManagedBy(m manager.Manager) *Builder { // ForInput represents the information set by For method. type ForInput struct { - object client.Object - predicates []predicate.Predicate - err error + object client.Object + predicates []predicate.Predicate + err error + conditionallyRun bool + waitTime time.Duration } // For defines the type of Object being *reconciled*, and configures the ControllerManagedBy to respond to create / delete / @@ -256,7 +260,34 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } ctrlOptions.Log = ctrlOptions.Log.WithValues("reconcilerGroup", gvk.Group, "reconcilerKind", gvk.Kind) - // Build the controller and return. - blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions) - return err + // Build the base controller + baseController, err := controller.NewUnmanaged(blder.getControllerName(gvk), blder.mgr, ctrlOptions) + if err != nil { + return err + } + + // Set the builder controller to either the base controller or wrapped as a ConditionalController. + var ctrl controller.Controller + if blder.forInput.conditionallyRun { + dc, err := discovery.NewDiscoveryClientForConfig(blder.mgr.GetConfig()) + if err != nil { + return err + } + sc := baseController.(controller.StoppableController) + sc.SaveWatches() + ctrl = &controller.ConditionalController{ + Cache: blder.mgr.GetCache(), + ConditionalOn: blder.forInput.object, + Controller: sc, + DiscoveryClient: dc, + Scheme: blder.mgr.GetScheme(), + WaitTime: blder.forInput.waitTime, + } + + } else { + ctrl = baseController + } + blder.ctrl = ctrl + + return blder.mgr.Add(ctrl) } diff --git a/pkg/builder/options.go b/pkg/builder/options.go index edd5d0156b..c80b57ef6f 100644 --- a/pkg/builder/options.go +++ b/pkg/builder/options.go @@ -17,9 +17,13 @@ limitations under the License. package builder import ( + "time" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) +const defaultWaitTime = time.Minute + // {{{ "Functional" Option Interfaces // ForOption is some configuration that modifies options for a For request. @@ -75,4 +79,23 @@ var _ ForOption = &Predicates{} var _ OwnsOption = &Predicates{} var _ WatchesOption = &Predicates{} +// ConditionallyRun runs the controller +// condtionally on the existence of the forInput object +// in the cluster's discovery doc, letting you start a +// controller manager for a CRD not yet installed on the cluster. +type ConditionallyRun struct { + WaitTime time.Duration +} + +// ApplyToFor applies this configuration to the give forInput options, +// setting the waitTime to the default wait time if it is unset. +func (w ConditionallyRun) ApplyToFor(opts *ForInput) { + opts.conditionallyRun = true + if w.WaitTime == time.Duration(0) { + opts.waitTime = defaultWaitTime + } else { + opts.waitTime = w.WaitTime + } +} + // }}} diff --git a/pkg/controller/conditional_controller.go b/pkg/controller/conditional_controller.go new file mode 100644 index 0000000000..cc92f78bc3 --- /dev/null +++ b/pkg/controller/conditional_controller.go @@ -0,0 +1,173 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// TODO: Comment everything +// ConditionalController is a helper that wraps a controller that +// can be stopped and restarted (StoppableController). +// +// Unique to the ConditionalController is that only runs the underlying controller +// when the object that controller watches (ConditionalOn), is installed in the cluster. +// +// Otherwise it will wait and periodically (WaitTime) check the discovery doc for +// existence of the ConditionalOn, starting, stopping, and restarting the controller +// as necessary based on the presence/absence of ConditionalOn. +type ConditionalController struct { + // Controller is the underlying controller that contains the Start() + // to be ran when ConditionalOn exists in the cluster. + Controller StoppableController + + // Cache is the manager's cache that must have the ConditionalOn object + // removed upon stopping the Controller. + Cache cache.Cache + + // ConditionalOn is the object being controlled by the Controller + // and for whose existence in the cluster/discover doc is required in order + // for the Controller to be running. + ConditionalOn runtime.Object + + // DiscoveryClient is used to query the discover doc for the existence + // of the ConditionalOn in the cluster. + DiscoveryClient *discovery.DiscoveryClient + + // Scheme helps convert between gvk and object. + Scheme *runtime.Scheme + + // WaitTime is how long to wait before rechecking the discovery doc. + WaitTime time.Duration +} + +// StoppableController is a wrapper around Controller providing extra methods +// that allow for running the controller multiple times. +type StoppableController interface { + Controller + + // ResetStart sets Started to false to enable running Start on the controller again. + ResetStart() + + // SaveWatches indicates that watches should not be cleared when the controller is stopped. + SaveWatches() +} + +// Reconcile implements the Controller interface. +func (c *ConditionalController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + return c.Controller.Reconcile(ctx, req) +} + +// Watch implements the Controller interface. +func (c *ConditionalController) Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error { + return c.Controller.Watch(src, eventhandler, predicates...) +} + +// GetLogger returns this controller's logger. +func (c *ConditionalController) GetLogger() logr.Logger { + return c.Controller.GetLogger() +} + +// Start condtionally runs the underlying controller based on the existence of the ConditionalOn in the cluster. +// In it's absence it waits for WaitTime before checking the discovery doc again. +func (c *ConditionalController) Start(ctx context.Context) error { + prevInstalled := false + curInstalled := false + errChan := make(chan error) + var mergeCtx context.Context + var cancel context.CancelFunc + //var presentStop chan struct{} + for { + select { + case err := <-errChan: + return err + case <-ctx.Done(): + return nil + case <-time.After(c.WaitTime): + gvk, err := apiutil.GVKForObject(c.ConditionalOn, c.Scheme) + if err != nil { + break + } + resources, err := c.DiscoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String()) + if err != nil { + curInstalled = false + } else { + curInstalled = false + for _, res := range resources.APIResources { + if res.Kind == gvk.Kind { + curInstalled = true + } + } + } + if !prevInstalled && curInstalled { + // Going from not installed -> installed. + // Start the runnable. + //presentStop = make(chan struct{}) + //mergedStop := mergeChan(presentStop, stop) + mergeCtx, cancel = context.WithCancel(ctx) + prevInstalled = true + go func() { + if err := c.Controller.Start(mergeCtx); err != nil { + errChan <- err + } + }() + } else if prevInstalled && !curInstalled { + // Going from installed -> not installed. + // Stop the runnable and remove the obj's informer from the cache. + // It's safe to remove the obj's informer because anything that is + // using it's informer will no longer work because the obj has been + // uninstalled from the cluster. + c.Controller.ResetStart() + c.Controller.SaveWatches() + //close(presentStop) + cancel() + // if we don't sleep, the cache will remove before the cancel propagates to remove the event handler + // TODO: Fix + time.Sleep(time.Second) + if err := c.Cache.Remove(ctx, c.ConditionalOn); err != nil { + return err + } + prevInstalled = false + } + } + } + +} + +// mergeChan return channel fires when either channel a or channel b is fired. +func mergeChan(a, b <-chan struct{}) chan struct{} { + out := make(chan struct{}) + go func() { + defer close(out) + select { + case <-a: + case <-b: + } + }() + return out +} diff --git a/pkg/controller/conditional_controller_test.go b/pkg/controller/conditional_controller_test.go new file mode 100644 index 0000000000..457fefa69c --- /dev/null +++ b/pkg/controller/conditional_controller_test.go @@ -0,0 +1,184 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + "context" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/testdata/foo" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/handler" + intctl "sigs.k8s.io/controller-runtime/pkg/internal/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var _ = Describe("controller.ConditionalController", func() { + var stop chan struct{} + + rec := reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + }) + BeforeEach(func() { + stop = make(chan struct{}) + }) + + AfterEach(func() { + close(stop) + }) + + Describe("Start", func() { + + It("should run manager for conditional runnables when corresponding CRD is not installed, installed, uninstalled and reinstalled", func(done Done) { + // initinalize scheme, crdOpts + s := runtime.NewScheme() + f := &foo.Foo{} + options := manager.Options{} + s.AddKnownTypeWithName(f.GroupVersionKind(), f) + options.Scheme = s + crdPath := filepath.Join(".", "testdata", "crds", "foocrd.yaml") + crdOpts := envtest.CRDInstallOptions{ + Paths: []string{crdPath}, + } + + // create manager + m, err := manager.New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + + // create conditional controller, add it to the manager + var conditionalOn runtime.Object + conditionalOn = &foo.Foo{} + runCh := make(chan int) + cache := m.GetCache() + ctrl, err := controller.NewUnmanaged("testController", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + internalCtrl, ok := ctrl.(*intctl.Controller) + Expect(ok).To(BeTrue()) + stopCtrl := &fakeStoppableController{ + controller: internalCtrl, + runCh: runCh, + } + + dc := discovery.NewDiscoveryClientForConfigOrDie(cfg) + condCtrl := &controller.ConditionalController{ + Controller: stopCtrl, + Cache: cache, + ConditionalOn: conditionalOn, + DiscoveryClient: dc, + Scheme: m.GetScheme(), + WaitTime: 20 * time.Millisecond, + } + Expect(m.Add(condCtrl)).NotTo(HaveOccurred()) + + // start the manager in a separate go routine + mgrStop := make(chan struct{}) + testLoopStart := make(chan struct{}) + go func() { + defer GinkgoRecover() + close(testLoopStart) + Expect(m.Start(mgrStop)).NotTo(HaveOccurred()) + close(runCh) + }() + + // run the test go routine to iterate through the situations where + // the CRD is + // 0) not installed + // 1) installed + // 2) uninstalled + // 3) reinstalled + // 4) uninstalled for test cleanup + testLoopDone := make(chan struct{}) + fakeCtrl, ok := condCtrl.Controller.(*fakeStoppableController) + Expect(ok).To(BeTrue()) + go func() { + defer GinkgoRecover() + <-testLoopStart + for i := 0; i < 5; i++ { + if i%2 == 1 { + // install CRD + crds, err := envtest.InstallCRDs(cfg, crdOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crds)).To(Equal(1)) + } else if i > 0 { + // uninstall CRD + err = envtest.UninstallCRDs(cfg, crdOpts) + Expect(err).NotTo(HaveOccurred()) + } + select { + case <-runCh: + //Expect(i % 2).To(Equal(1)) + // CRD is installed + case <-time.After(60 * time.Millisecond): + //Expect(i % 2).To(Equal(0)) + // CRD is NOT installed + fakeCtrl.noStartCount++ + } + } + close(mgrStop) + close(testLoopDone) + }() + <-testLoopDone + Expect(fakeCtrl.startCount).To(Equal(2)) + Expect(fakeCtrl.noStartCount).To(Equal(3)) + close(done) + }) + + }) +}) + +type fakeStoppableController struct { + controller controller.StoppableController + startCount int + noStartCount int + runCh chan int +} + +// Methods for fakeStoppableController to conform to the StoppableController interface +func (f *fakeStoppableController) ResetStart() { + f.controller.ResetStart() +} + +func (f *fakeStoppableController) SaveWatches() { + f.controller.SaveWatches() +} + +func (f *fakeStoppableController) Start(s <-chan struct{}) error { + //fmt.Println("bumping start") + f.startCount++ + f.runCh <- 1 + return nil +} + +func (f *fakeStoppableController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + return f.controller.Reconcile(ctx, req) +} + +func (f *fakeStoppableController) Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error { + return f.controller.Watch(src, eventhandler, predicates...) +} diff --git a/pkg/controller/testdata/crds/foocrd.yaml b/pkg/controller/testdata/crds/foocrd.yaml new file mode 100644 index 0000000000..e2bddbc528 --- /dev/null +++ b/pkg/controller/testdata/crds/foocrd.yaml @@ -0,0 +1,17 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.bar.example.com +spec: + group: bar.example.com + names: + kind: Foo + plural: foos + scope: Namespaced + versions: + - name: "v1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/controller/testdata/foo/foo_types.go b/pkg/controller/testdata/foo/foo_types.go new file mode 100644 index 0000000000..b01d31c038 --- /dev/null +++ b/pkg/controller/testdata/foo/foo_types.go @@ -0,0 +1,36 @@ +package foo + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type FooSpec struct{} +type FooStatus struct{} + +type Foo struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FooSpec `json:"spec,omitempty"` + Status FooStatus `json:"status,omitempty"` +} + +func (f *Foo) DeepCopyObject() runtime.Object { + return nil +} + +func (f *Foo) SetGroupVersionKind(kind schema.GroupVersionKind) {} +func (f *Foo) GroupVersionKind() schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: "bar.example.com", + Version: "v1", + Kind: "Foo", + } +} + +func (f *Foo) GetObjectKind() schema.ObjectKind { + return f + +} From 35b4547b5510afa7fe0aafc08a393c1164be7489 Mon Sep 17 00:00:00 2001 From: Kevin Delgado Date: Tue, 20 Oct 2020 01:41:17 +0000 Subject: [PATCH 6/6] RunWithStopOptions first pass working with client-go changes --- go.sum | 1 + pkg/cache/cache.go | 15 +---------- pkg/cache/internal/counting_informer.go | 33 ++++++++++++----------- pkg/cache/internal/informers_map.go | 36 ++++++++++++++----------- pkg/cache/multi_namespace_cache.go | 3 +-- 5 files changed, 41 insertions(+), 47 deletions(-) diff --git a/go.sum b/go.sum index 223d3a68e8..1a2dd48eb0 100644 --- a/go.sum +++ b/go.sum @@ -478,6 +478,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index fc95933065..833e80c94b 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -98,22 +98,9 @@ type Informer interface { // RunWithStopOptions runs the informer and provides options to be checked that // would indicate under what conditions the informer should stop. - RunWithStopOptions(stopOptions internal.StopOptions) internal.StopReason + RunWithStopOptions(stopOptions toolscache.StopOptions) toolscache.StopReason } -// StopOptions let the caller specify which conditions to stop the informer. -type StopOptions struct { - // StopChannel stops the Informer when it receives a close signal. - StopChannel <-chan struct{} - - // OnListError inspects errors returned from the infromer's underlying refloector, - // and based on the error determines whether or not to stop the informer. - OnListError func(error) bool -} - -// StopReason is a custom typed error that indicates how the informer was stopped. -type StopReason error - // Options are the optional arguments for creating a new InformersMap object type Options struct { // Scheme is the scheme to use for mapping objects to GroupVersionKinds diff --git a/pkg/cache/internal/counting_informer.go b/pkg/cache/internal/counting_informer.go index 297f55adc0..5471d15614 100644 --- a/pkg/cache/internal/counting_informer.go +++ b/pkg/cache/internal/counting_informer.go @@ -13,7 +13,7 @@ type CountingInformer interface { cache.SharedIndexInformer CountEventHandlers() int RemoveEventHandler(id int) error - RunWithStopOptions(stopOptions StopOptions) StopReason + RunWithStopOptions(stopOptions cache.StopOptions) cache.StopReason } // HandlerCountingInformer implements the CountingInformer. @@ -86,20 +86,21 @@ func (i *HandlerCountingInformer) Run(stopCh <-chan struct{}) { i.informer.Run(stopCh) } -func (i *HandlerCountingInformer) RunWithStopOptions(stopOptions StopOptions) StopReason { - i.Run(stopOptions.StopChannel) - return nil -} - -// StopOptions let the caller specify which conditions to stop the informer. -type StopOptions struct { - // StopChannel stops the Informer when it receives a close signal. - StopChannel <-chan struct{} - - // OnListError inspects errors returned from the infromer's underlying refloector, - // and based on the error determines whether or not to stop the informer. - OnListError func(error) bool +func (i *HandlerCountingInformer) RunWithStopOptions(stopOptions cache.StopOptions) cache.StopReason { + return i.informer.RunWithStopOptions(stopOptions) } -// StopReason is a custom typed error that indicates how the informer was stopped. -type StopReason error +//// StopOptions let the caller specify which conditions to stop the informer. +//type StopOptions struct { +// // StopChannel stops the Informer when it receives a close signal. +// StopChannel <-chan struct{} +// +// Cancel context.CancelFunc +// +// // OnListError inspects errors returned from the infromer's underlying refloector, +// // and based on the error determines whether or not to stop the informer. +// OnListError func(error) bool +//} +// +//// StopReason is a custom typed error that indicates how the informer was stopped. +//type StopReason error diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go index 0daaf23f02..4e9ce9f100 100644 --- a/pkg/cache/internal/informers_map.go +++ b/pkg/cache/internal/informers_map.go @@ -128,18 +128,23 @@ type specificInformersMap struct { // Blocks until the informer stops. The informer can be stopped // either individually (via the entry's stop channel) or globally // via the provided stop argument. -func (e *MapEntry) Start(stop <-chan struct{}) { - // Stop on either the whole map stopping or just this informer being removed. - internalStop, cancel := anyOf(stop, e.stop) - defer cancel() - e.Informer.Run(internalStop) -} - -func (e *MapEntry) StartWithStopOptions(stopOptions StopOptions) { +//func (e *MapEntry) Start(stop <-chan struct{}) { +// // Stop on either the whole map stopping or just this informer being removed. +// internalStop, cancel := anyOf(stop, e.stop) +// defer cancel() +// e.Informer.Run(internalStop) +//} + +func (e *MapEntry) StartWithStopOptions(stopOptions cache.StopOptions) { // Stop on either the whole map stopping or just this informer being removed. internalStop, cancel := anyOf(stopOptions.StopChannel, e.stop) stopOptions.StopChannel = internalStop - defer cancel() + stopOptions.Cancel = cancel + stopOptions.OnListError = func(err error) bool { + fmt.Println("OnListError") + return true + } + //defer cancel() e.Informer.RunWithStopOptions(stopOptions) } @@ -157,7 +162,7 @@ func (ip *specificInformersMap) Start(ctx context.Context) { // Start each informer for _, entry := range ip.informersByGVK { //go entry.Start(ctx.Done()) - go entry.StartWithStopOptions(StopOptions{ + go entry.StartWithStopOptions(cache.StopOptions{ StopChannel: ctx.Done(), }) } @@ -172,7 +177,7 @@ func (ip *specificInformersMap) Start(ctx context.Context) { // StartWithStopOptions exposes a way to start an informer with user defined stopOptions // We would plumb stopOptions from the builder (where the user would define them), down here through to the // informer. -func (ip *specificInformersMap) StartWithStopOptions(ctx context.Context, stopOptions StopOptions) { +func (ip *specificInformersMap) StartWithStopOptions(ctx context.Context, stopOptions cache.StopOptions) { // TODO: implement once SharedIndexInformer in client-go supports RunWithStopOptions } @@ -259,10 +264,11 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob // TODO(seans): write thorough tests and document what happens here - can you add indexers? // can you add eventhandlers? if ip.started { - go i.Start(ip.stop) - //go i.Start(StopOptions{ - // StopChannel: ip.stop, - //}) + fmt.Println("HUH ALREADY STARTED??") + //go i.Start(ip.stop) + go i.StartWithStopOptions(cache.StopOptions{ + StopChannel: ip.stop, + }) } return i, ip.started, nil } diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index 392a3a7777..c7c87cd754 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" - "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -196,7 +195,7 @@ type multiNamespaceInformer struct { var _ Informer = &multiNamespaceInformer{} -func (i *multiNamespaceInformer) RunWithStopOptions(stopOptions internal.StopOptions) internal.StopReason { +func (i *multiNamespaceInformer) RunWithStopOptions(stopOptions toolscache.StopOptions) toolscache.StopReason { // TODO: don't leak goro - collect them with a errgroup or waitgroup for _, informer := range i.namespaceToInformer { go informer.RunWithStopOptions(stopOptions)