Merge pull request #46 from flanksource/moshloop

add namespaced tolerations

.github/workflows/test.yml

@@ -1,6 +1,5 @@
 name: Test
 on:
-  push:
   pull_request:
 jobs:
   build:

Makefile

@@ -33,11 +33,13 @@ all: manager
 test: fmt vet
 	go test ./... -coverprofile cover.out -v
 
-e2e: fmt vet
-	TEST_E2E=true go test ./... -coverprofile cover.out -v
+e2e:
+	TEST_E2E=true go test ./test/... -coverprofile cover.out -v  -ginkgo.v
 
 # Build manager binary
-manager: fmt vet
+manager: fmt vet build
+
+build:
 	go build -o bin/manager cmd/manager/main.go
 
 # Build manager binary

README.md

@@ -2,39 +2,165 @@
 
 Platform Operator is Kubernetes operator designed to be run in a multi-tenanted environment.
 
-Current features:
+### Namespaced Tolerations
 
-* Auto-Delete: cleanup namespaces after a certain expiry period by labeleling the namespace with `auto-delete`.
-* ClusterResourceQuota: allows quotas to be enforced across the entire cluster.
+Applies tolerations to all pods in a namespace, based on annotations on the namespace
 
-## Install
+e.g. using`--enable-pod-mutations=true --namespace-tolerations-prefix=tolerations`
 
-1. Generate the YAML manifests containing all the resources (CRDs, namespaces, Deployment, etc...)
+```yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dedicate-to-node-group-b
+  annotations:
+    tolerations/node-group: b
+```
+
+Will then result in all pods created in that namespace receiving a toleration of:
+
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+   tolerations:
+     key: node-group
+     value: b
+     effect: NoSchedule
+```
+
+### Namespace Annotation Defaults
+
+e.g. with `--enable-pod-mutations=true --annotations=co.elastic`
+
+```yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dedicate-to-node-group-b
+  annotations:
+    co.elastic.logs/enabled: true
+```
+
+Will then result in all pods created in that namespace defaulting to:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    co.elastic.logs/enabled: true
+```
+
+### Registry Defaults
+
+e.g. with `--enable-pod-mutations=true --default-registry-prefix==registry.corp`
+
+When creating a pod with a `busybox:latest`  such as:
+
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+		- image: busybox:latest
+```
+
+It will get mutated to:
+
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+		- image: registry.corp/busybox:latest
+```
 
-```shell
-make generate
+To prevent some images from being prefixed use `--registry-whitelist` e.g.  `--registry-whitelist=k8s.gcr.io`
+
+Add a default image pull secret to all pods using `--default-image-pull-secret`
+
+### Auto Delete
+
+- `--cleanup=true` - Delete resources with `auto-delete` annotations specified in duration from creation
+  - `--cleanup-interval` - Interval to check for resources to cleanup
+
+```yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pr-workflow-123
+  annotations:
+     auto-delete: 24h # delete this namespace 24h after creation
 ```
 
-This command will create the file [manifests.yaml](config/deploy/manifests.yaml). Don't make manual changes to this file.
+### Cluster Resource Quotas
+
+- `--enable-cluster-resource-quota` - Allow resource quotas to be defined at cluster level
 
-2. Deploy the generated configuration in the cluster:
+```yaml
+apiVersion: platform.flanksource.com/v1
+kind: ClusterResourceQuota
+metadata:
+  name: dynamic-pr-compute-resources
+spec:
+  matchLabels:
+    owner: dynamic-pr
+  hard:
+    requests.cpu: "1"
+    requests.memory: 1Gi
+    limits.cpu: "1"
+    limits.memory: 1Gi
+    pods: "10"
+    services.loadbalancers: "0"
+    services.nodeports: "0"
 
-```shell
-make deploy
-kubectl apply -f config/deploy/manifests.yaml
-namespace/flanksource-system created
-customresourcedefinition.apiextensions.k8s.io/clusterresourcequotas.platform.flanksource.com created
-validatingwebhookconfiguration.admissionregistration.k8s.io/flanksource-validating-webhook-configuration created
-role.rbac.authorization.k8s.io/flanksource-leader-election created
-clusterrole.rbac.authorization.k8s.io/flanksource-clusterresourcequota-editor created
-clusterrole.rbac.authorization.k8s.io/flanksource-clusterresourcequota-viewer created
-clusterrole.rbac.authorization.k8s.io/flanksource-manager created
-rolebinding.rbac.authorization.k8s.io/flanksource-leader-election created
-clusterrolebinding.rbac.authorization.k8s.io/flanksource-manager created
-service/flanksource-webhook-service created
-deployment.apps/flanksource-controller-manager created
-certificate.cert-manager.io/flanksource-serving-cert configured
-issuer.cert-manager.io/flanksource-selfsigned-issuer configured
 ```
 
-This command use `kustomize` to build the manifests. Once ready the manifests are applied to the cluster and the operator starts.
+
+
+### Ingress SSO
+
+Depends on karina ingress as is normally deployed only via karina using:
+
+`karina.yml`
+
+```yaml
+domain: ACMP.corp
+ldap:
+	....
+dex:
+	version: v2.27.0
+oauth2Proxy:
+  version: v6.1.1
+platformOperator:
+  version: v0.6.0
+```
+
+- `--enable-ingress-sso` enable ingress SSO using `platform.flanksource.com/restrict-to-groups` annotations
+  - `--oauth2-proxy-service-name`
+  - `--oauth2-proxy-service-namespace`
+  - `--domain`
+
+>  See https://karina.docs.flanksource.com/admin-guide/ingress/ for more details on how to configure the ingress, before using the platform-operator.
+
+Once installed ingresses can be restricted using:
+
+```yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: podinfo-ing
+  namespace: default
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    platform.flanksource.com/restrict-to-groups: ADMINS
+```
+
+
+
+| Annotation                                             | Description                                                  |
+| ------------------------------------------------------ | ------------------------------------------------------------ |
+| `platform.flanksource.com/restrict-to-groups`          | A semi-colon delimited list of LDAP groups to restrict an ingress to |
+| `platform.flanksource.com/extra-configuration-snippet` | Any additional nginx snippets to apply to the location       |
+| `platform.flanksource.com/pass-auth-headers`           | Specify `true` to pass authentication headers all the way through to the ingress upstream |

cmd/manager/main.go

@@ -29,7 +29,7 @@ import (
 	"github.com/flanksource/platform-operator/pkg/controllers/cleanup"
 	"github.com/flanksource/platform-operator/pkg/controllers/clusterresourcequota"
 	"github.com/flanksource/platform-operator/pkg/controllers/ingress"
-	"github.com/flanksource/platform-operator/pkg/controllers/podannotator"
+	"github.com/flanksource/platform-operator/pkg/controllers/pod"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -55,13 +55,17 @@ func init() {
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
+
 	var cleanupInterval, annotationInterval time.Duration
 	var enableClusterResourceQuota bool
+	var ingressSSO bool
 	var oauth2ProxySvcName string
 	var oauth2ProxySvcNamespace string
 	var domain string
+
 	var registryWhitelist string
 	var annotations string
+	var podMutator bool
 	cfg := platformv1.PodMutaterConfig{}
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
@@ -72,16 +76,20 @@ func main() {
 	flag.DurationVar(&cleanupInterval, "cleanup-interval", 10*time.Minute, "Frequency at which the cleanup controller runs.")
 	flag.DurationVar(&annotationInterval, "annotation-interval", 10*time.Minute, "Frequency at which the annotation controller runs.")
 
-	flag.StringVar(&annotations, "annotations", "", "Annotations pods inherit from parent namespace")
 	flag.BoolVar(&enableClusterResourceQuota, "enable-cluster-resource-quota", true, "Enable/Disable cluster resource quota")
 
+	flag.BoolVar(&ingressSSO, "enable-ingress-sso", false, "Enable ingress mutation hook for restrict-to-groups SSO")
 	flag.StringVar(&oauth2ProxySvcName, "oauth2-proxy-service-name", "", "Name of oauth2-proxy service")
 	flag.StringVar(&oauth2ProxySvcNamespace, "oauth2-proxy-service-namespace", "", "Name of oauth2-proxy service namespace")
-	flag.StringVar(&cfg.DefaultRegistryPrefix, "default-registry-prefix", "", "A default registry prefix path to apply to all pods")
-	flag.StringVar(&cfg.DefaultImagePullSecret, "default-image-pull-secret", "", "Default dmage pull secret to apply to all pods")
-	flag.StringVar(&registryWhitelist, "registry-whitelist", "", "A list of image prefixes to ignore")
 	flag.StringVar(&domain, "domain", "", "Domain used by platform")
 
+	flag.BoolVar(&podMutator, "enable-pod-mutations", true, "Enable pod mutating webhooks")
+
+	flag.StringVar(&annotations, "annotations", "", "Annotations pods inherit from parent namespace")
+	flag.StringVar(&cfg.DefaultRegistryPrefix, "default-registry-prefix", "", "A default registry prefix path to apply to all pods")
+	flag.StringVar(&cfg.DefaultImagePullSecret, "default-image-pull-secret", "", "A default image pull secret to apply to all pods")
+	flag.StringVar(&registryWhitelist, "registry-whitelist", "", "A list of image prefixes to ignore")
+	flag.StringVar(&cfg.TolerationsPrefix, "namespace-tolerations-prefix", "tolerations/", "A prefix for namespace level annotations that should be applied as tolerations on pods")
 	flag.Parse()
 
 	cfg.Annotations = strings.Split(annotations, ",")
@@ -102,7 +110,11 @@ func main() {
 		os.Exit(1)
 	}
 
-	// TODO(mazzy89): Make the adding of controllers more dynamic
+	// Setup webhooks
+	setupLog.Info("setting up webhook server")
+	hookServer := mgr.GetWebhookServer()
+
+	mtx := &sync.Mutex{}
 
 	if err := cleanup.Add(mgr, cleanupInterval); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Cleanup")
@@ -114,32 +126,27 @@ func main() {
 			setupLog.Error(err, "unable to create controller", "controller", "ClusterResourceQuota")
 			os.Exit(1)
 		}
+		hookServer.Register("/validate-clusterresourcequota-platform-flanksource-com-v1", clusterresourcequota.NewClusterResourceQuotaValidatingWebhook(mgr.GetClient(), mtx, enableClusterResourceQuota))
+		hookServer.Register("/validate-resourcequota-v1", clusterresourcequota.NewResourceQuotaValidatingWebhook(mgr.GetClient(), mtx, enableClusterResourceQuota))
+
 	}
 
-	if err := podannotator.Add(mgr, annotationInterval, cfg); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "PodAnnotator")
-		os.Exit(1)
+	if podMutator {
+		if err := pod.Add(mgr, annotationInterval, cfg); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "PodAnnotator")
+			os.Exit(1)
+		}
+		hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: pod.NewMutatingWebhook(mgr.GetClient(), cfg)})
 	}
 
-	if oauth2ProxySvcName != "" && oauth2ProxySvcNamespace != "" {
+	if ingressSSO {
 		if err := ingress.Add(mgr, annotationInterval, oauth2ProxySvcName, oauth2ProxySvcNamespace, domain); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "IngressAnnotator")
 			os.Exit(1)
 		}
+		hookServer.Register("/mutate-v1-ingress", &webhook.Admission{Handler: ingress.NewMutatingWebhook(mgr.GetClient(), oauth2ProxySvcName, oauth2ProxySvcNamespace, domain)})
 	}
 
-	// Setup webhooks
-	setupLog.Info("setting up webhook server")
-	hookServer := mgr.GetWebhookServer()
-
-	mtx := &sync.Mutex{}
-
-	setupLog.Info("registering webhooks to the webhook server")
-	hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: platformv1.PodAnnotatorMutateWebhook(mgr.GetClient(), cfg)})
-	hookServer.Register("/mutate-v1-ingress", &webhook.Admission{Handler: platformv1.IngressAnnotatorMutateWebhook(mgr.GetClient(), oauth2ProxySvcName, oauth2ProxySvcNamespace, domain)})
-	hookServer.Register("/validate-clusterresourcequota-platform-flanksource-com-v1", platformv1.ClusterResourceQuotaValidatingWebhook(mtx, enableClusterResourceQuota))
-	hookServer.Register("/validate-resourcequota-v1", platformv1.ResourceQuotaValidatingWebhook(mtx, enableClusterResourceQuota))
-
 	// +kubebuilder:scaffold:builder
 
 	go func() {