Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Step 5 – generic controplane: split aggregator API priorities #124613

Merged
merged 2 commits into from
Apr 30, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
91 changes: 31 additions & 60 deletions cmd/kube-apiserver/app/aggregator.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,14 @@ import (
apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
"k8s.io/kube-aggregator/pkg/controllers/autoregister"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
controlplaneapiserveroptions "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/controller/crdregistration"
)

func createAggregatorConfig(
kubeAPIServerConfig genericapiserver.Config,
commandOptions controlplaneapiserver.CompletedOptions,
commandOptions controlplaneapiserveroptions.CompletedOptions,
externalInformers kubeexternalinformers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
proxyTransport *http.Transport,
Expand Down Expand Up @@ -156,7 +157,7 @@ func createAggregatorServer(aggregatorConfig aggregatorapiserver.CompletedConfig
// Imbue all builtin group-priorities onto the aggregated discovery
if aggregatorConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil {
for gv, entry := range apiVersionPriorities {
aggregatorConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.group), int(entry.version))
aggregatorConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.Group), int(entry.Version))
}
}

Expand Down Expand Up @@ -210,8 +211,8 @@ func makeAPIService(gv schema.GroupVersion) *v1.APIService {
Spec: v1.APIServiceSpec{
Group: gv.Group,
Version: gv.Version,
GroupPriorityMinimum: apiServicePriority.group,
VersionPriority: apiServicePriority.version,
GroupPriorityMinimum: apiServicePriority.Group,
VersionPriority: apiServicePriority.Version,
},
}
}
Expand Down Expand Up @@ -255,69 +256,32 @@ func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIServic
})
}

// priority defines group priority that is used in discovery. This controls
// group position in the kubectl output.
type priority struct {
// group indicates the order of the group relative to other groups.
group int32
// version indicates the relative order of the version inside of its group.
version int32
}

// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers
// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed.
// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated.
// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage.
var apiVersionPriorities = map[schema.GroupVersion]priority{
{Group: "", Version: "v1"}: {group: 18000, version: 1},
var apiVersionPriorities = merge(controlplaneapiserver.DefaultGenericAPIServicePriorities(), map[schema.GroupVersion]controlplaneapiserver.APIServicePriority{
{Group: "", Version: "v1"}: {Group: 18000, Version: 1},
// to my knowledge, nothing below here collides
{Group: "apps", Version: "v1"}: {group: 17800, version: 15},
{Group: "events.k8s.io", Version: "v1"}: {group: 17750, version: 15},
{Group: "events.k8s.io", Version: "v1beta1"}: {group: 17750, version: 5},
{Group: "authentication.k8s.io", Version: "v1"}: {group: 17700, version: 15},
{Group: "authentication.k8s.io", Version: "v1beta1"}: {group: 17700, version: 9},
{Group: "authentication.k8s.io", Version: "v1alpha1"}: {group: 17700, version: 1},
{Group: "authorization.k8s.io", Version: "v1"}: {group: 17600, version: 15},
{Group: "autoscaling", Version: "v1"}: {group: 17500, version: 15},
{Group: "autoscaling", Version: "v2"}: {group: 17500, version: 30},
{Group: "autoscaling", Version: "v2beta1"}: {group: 17500, version: 9},
{Group: "autoscaling", Version: "v2beta2"}: {group: 17500, version: 1},
{Group: "batch", Version: "v1"}: {group: 17400, version: 15},
{Group: "batch", Version: "v1beta1"}: {group: 17400, version: 9},
{Group: "batch", Version: "v2alpha1"}: {group: 17400, version: 9},
{Group: "certificates.k8s.io", Version: "v1"}: {group: 17300, version: 15},
{Group: "certificates.k8s.io", Version: "v1alpha1"}: {group: 17300, version: 1},
{Group: "networking.k8s.io", Version: "v1"}: {group: 17200, version: 15},
{Group: "networking.k8s.io", Version: "v1alpha1"}: {group: 17200, version: 1},
{Group: "policy", Version: "v1"}: {group: 17100, version: 15},
{Group: "policy", Version: "v1beta1"}: {group: 17100, version: 9},
{Group: "rbac.authorization.k8s.io", Version: "v1"}: {group: 17000, version: 15},
{Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15},
{Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9},
{Group: "storage.k8s.io", Version: "v1alpha1"}: {group: 16800, version: 1},
{Group: "apiextensions.k8s.io", Version: "v1"}: {group: 16700, version: 15},
{Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15},
{Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9},
{Group: "scheduling.k8s.io", Version: "v1"}: {group: 16600, version: 15},
{Group: "coordination.k8s.io", Version: "v1"}: {group: 16500, version: 15},
{Group: "node.k8s.io", Version: "v1"}: {group: 16300, version: 15},
{Group: "node.k8s.io", Version: "v1alpha1"}: {group: 16300, version: 1},
{Group: "node.k8s.io", Version: "v1beta1"}: {group: 16300, version: 9},
{Group: "discovery.k8s.io", Version: "v1"}: {group: 16200, version: 15},
{Group: "discovery.k8s.io", Version: "v1beta1"}: {group: 16200, version: 12},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}: {group: 16100, version: 21},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}: {group: 16100, version: 18},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}: {group: 16100, version: 15},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}: {group: 16100, version: 12},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16100, version: 9},
{Group: "internal.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16000, version: 9},
{Group: "resource.k8s.io", Version: "v1alpha2"}: {group: 15900, version: 9},
{Group: "storagemigration.k8s.io", Version: "v1alpha1"}: {group: 15800, version: 9},
{Group: "apps", Version: "v1"}: {Group: 17800, Version: 15},
{Group: "batch", Version: "v1"}: {Group: 17400, Version: 15},
{Group: "batch", Version: "v1beta1"}: {Group: 17400, Version: 9},
{Group: "batch", Version: "v2alpha1"}: {Group: 17400, Version: 9},
{Group: "networking.k8s.io", Version: "v1"}: {Group: 17200, Version: 15},
{Group: "networking.k8s.io", Version: "v1alpha1"}: {Group: 17200, Version: 1},
{Group: "policy", Version: "v1"}: {Group: 17100, Version: 15},
{Group: "policy", Version: "v1beta1"}: {Group: 17100, Version: 9},
{Group: "storage.k8s.io", Version: "v1"}: {Group: 16800, Version: 15},
{Group: "storage.k8s.io", Version: "v1beta1"}: {Group: 16800, Version: 9},
{Group: "storage.k8s.io", Version: "v1alpha1"}: {Group: 16800, Version: 1},
{Group: "scheduling.k8s.io", Version: "v1"}: {Group: 16600, Version: 15},
{Group: "node.k8s.io", Version: "v1"}: {Group: 16300, Version: 15},
{Group: "node.k8s.io", Version: "v1alpha1"}: {Group: 16300, Version: 1},
{Group: "node.k8s.io", Version: "v1beta1"}: {Group: 16300, Version: 9},
// Append a new group to the end of the list if unsure.
// You can use min(existing group)-100 as the initial value for a group.
// Version can be set to 9 (to have space around) for a new group.
}
})

func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService {
apiServices := []*v1.APIService{}
Expand Down Expand Up @@ -349,3 +313,10 @@ func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget,

return apiServices
}

func merge(a, b map[schema.GroupVersion]controlplaneapiserver.APIServicePriority) map[schema.GroupVersion]controlplaneapiserver.APIServicePriority {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

now it's not a problem, but I was first thinking that it could be problematic if this function would be passed some generic global variable, as this func mutates a. You indeed solved it with having a function returning a new variable every time, but just something to be careful about, unless this function first does a deep copy of a.

for k, v := range b {
a[k] = v
}
return a
}
73 changes: 73 additions & 0 deletions pkg/controlplane/apiserver/aggregator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
Copyright 2024 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
"k8s.io/apimachinery/pkg/runtime/schema"
)

// APIServicePriority defines group priority that is used in discovery. This controls
// group position in the kubectl output.
type APIServicePriority struct {
// Group indicates the order of the group relative to other groups.
Group int32
// Version indicates the relative order of the Version inside of its group.
Version int32
}

// DefaultGenericAPIServicePriorities returns the APIService priorities for generic APIs
func DefaultGenericAPIServicePriorities() map[schema.GroupVersion]APIServicePriority {
// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers
// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed.
// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated.
// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage.
return map[schema.GroupVersion]APIServicePriority{
{Group: "", Version: "v1"}: {Group: 18000, Version: 1},
// to my knowledge, nothing below here collides
{Group: "events.k8s.io", Version: "v1"}: {Group: 17750, Version: 15},
{Group: "events.k8s.io", Version: "v1beta1"}: {Group: 17750, Version: 5},
{Group: "authentication.k8s.io", Version: "v1"}: {Group: 17700, Version: 15},
{Group: "authentication.k8s.io", Version: "v1beta1"}: {Group: 17700, Version: 9},
{Group: "authentication.k8s.io", Version: "v1alpha1"}: {Group: 17700, Version: 1},
{Group: "authorization.k8s.io", Version: "v1"}: {Group: 17600, Version: 15},
{Group: "autoscaling", Version: "v1"}: {Group: 17500, Version: 15},
{Group: "autoscaling", Version: "v2"}: {Group: 17500, Version: 30},
{Group: "autoscaling", Version: "v2beta1"}: {Group: 17500, Version: 9},
{Group: "autoscaling", Version: "v2beta2"}: {Group: 17500, Version: 1},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason these are in the controlplane code and not in kube-apiserver?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

aren't these for kube-apiserver related aka workloads?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aren't they working against any resource with scale resource? Then they are generic.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I agree with you, looking into that group which resources it exposes. Updated the PR.

{Group: "certificates.k8s.io", Version: "v1"}: {Group: 17300, Version: 15},
{Group: "certificates.k8s.io", Version: "v1alpha1"}: {Group: 17300, Version: 1},
{Group: "rbac.authorization.k8s.io", Version: "v1"}: {Group: 17000, Version: 15},
{Group: "apiextensions.k8s.io", Version: "v1"}: {Group: 16700, Version: 15},
{Group: "admissionregistration.k8s.io", Version: "v1"}: {Group: 16700, Version: 15},
{Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {Group: 16700, Version: 12},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {Group: 16700, Version: 9},
{Group: "coordination.k8s.io", Version: "v1"}: {Group: 16500, Version: 15},
{Group: "discovery.k8s.io", Version: "v1"}: {Group: 16200, Version: 15},
{Group: "discovery.k8s.io", Version: "v1beta1"}: {Group: 16200, Version: 12},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1"}: {Group: 16100, Version: 21},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta3"}: {Group: 16100, Version: 18},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2"}: {Group: 16100, Version: 15},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}: {Group: 16100, Version: 12},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1"}: {Group: 16100, Version: 9},
{Group: "internal.apiserver.k8s.io", Version: "v1alpha1"}: {Group: 16000, Version: 9},
{Group: "resource.k8s.io", Version: "v1alpha2"}: {Group: 15900, Version: 9},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this really generic? This seems to be related to Dynamic Resource Allocation, which is pod-specific if I'm not mistaken?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question. I can see it has few types we might want to use in building some quota machinery:
https://pkg.go.dev/k8s.io/api@v0.30.0/resource/v1alpha2
but in general I think you right.

{Group: "storagemigration.k8s.io", Version: "v1alpha1"}: {Group: 15800, Version: 9},
// Append a new group to the end of the list if unsure.
// You can use min(existing group)-100 as the initial value for a group.
// Version can be set to 9 (to have space around) for a new group.
}
}