Add a flag for enabling pprof on the controller manager (#4449)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Makefile

@@ -6,7 +6,7 @@ endif
 DOCKER_USER ?= $(shell echo ${DOCKER_IMAGE_NAME} | cut -d / -f1)
 VERSION ?= dev
 COMMIT_SHA = $(shell git rev-parse HEAD)
-RUNNER_VERSION ?= 2.333.1
+RUNNER_VERSION ?= 2.334.0
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -84,6 +84,9 @@ spec:
         - "--listener-metrics-endpoint="
         - "--metrics-addr=0"
         {{- end }}
+        {{- if .Values.pprof.addr }}
+        - "--pprof-addr={{ .Values.pprof.addr }}"
+        {{- end }}
         {{- range .Values.flags.excludeLabelPropagationPrefixes }}
         - "--exclude-label-propagation-prefix={{ . }}"
         {{- end }}
@@ -93,14 +96,26 @@ spec:
         {{- with .Values.flags.k8sClientRateLimiterBurst }}
         - "--k8s-client-rate-limiter-burst={{ . }}"
         {{- end }}
+        {{- with .Values.flags.rateLimiter }}
+        {{- with .name }}
+        - "--workqueue-rate-limiter={{ . }}"
+        {{- end }}
+        {{- end }}
         command:
         - "/manager"
-        {{- with .Values.metrics }}
+        {{- if or .Values.metrics .Values.pprof.addr }}
         ports:
-        - containerPort: {{regexReplaceAll ":([0-9]+)" .controllerManagerAddr "${1}"}}
+        {{- end }}
+        {{- with .Values.metrics }}
+        - containerPort: {{ required "Values.metrics.controllerManagerAddr must end with a numeric port" (regexFind "[0-9]+$" .controllerManagerAddr) }}
           protocol: TCP
           name: metrics
         {{- end }}
+        {{- if .Values.pprof.addr }}
+        - containerPort: {{ required "Values.pprof.addr must end with a numeric port" (regexFind "[0-9]+$" .Values.pprof.addr) }}
+          protocol: TCP
+          name: pprof
+        {{- end }}
         env:
         - name: CONTROLLER_MANAGER_CONTAINER_IMAGE
           value: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

charts/gha-runner-scale-set-controller/values.yaml

@@ -94,6 +94,10 @@ priorityClassName: ""
 #   listenerAddr: ":8080"
 #   listenerEndpoint: "/metrics"
 
+## To enable pprof, uncomment the addr field below.
+pprof: {}
+#   addr: ":6060"
+
 flags:
   ## Log level can be set here with one of the following values: "debug", "info", "warn", "error".
   ## Defaults to "debug".
@@ -136,6 +140,13 @@ flags:
   # excludeLabelPropagationPrefixes:
   #   - "argocd.argoproj.io/instance"
 
+  ## Workqueue rate limiter configuration.
+  ## By default, controller-runtime uses a combined rate limiter with both a per-item
+  ## exponential backoff and an overall token bucket (10 QPS, 100 bucket size).
+  ## Valid names: "bucket_rate_limiter" (default), "typed_rate_limiter" (per-item only, no global token bucket).
+  # rateLimiter:
+  #   name: "bucket_rate_limiter"
+
 # Overrides the default `.Release.Namespace` for all resources in this chart.
 namespaceOverride: ""
 

charts/gha-runner-scale-set/templates/no_permission_serviceaccount.yaml

@@ -1,6 +1,6 @@
 {{- $hasCustomResourceMeta := (and .Values.resourceMeta .Values.resourceMeta.noPermissionServiceAccount) }}
 {{- $containerMode := .Values.containerMode }}
-{{- if and (ne $containerMode.type "kubernetes") (not .Values.template.spec.serviceAccountName) }}
+{{- if and (ne $containerMode.type "kubernetes") (ne $containerMode.type "kubernetes-novolume") (not .Values.template.spec.serviceAccountName) }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

charts/gha-runner-scale-set/tests/template_test.go

@@ -335,6 +335,46 @@ func TestTemplateRenderedSetServiceAccountToKubeNoVolumeMode(t *testing.T) {
 	assert.Equal(t, expectedServiceAccountName, ars.Annotations[actionsgithubcom.AnnotationKeyKubernetesModeServiceAccountName])
 }
 
+func TestTemplateRenderedNoPermissionServiceAccountNotRenderedInKubernetesModes(t *testing.T) {
+	t.Parallel()
+
+	for _, mode := range []string{"kubernetes", "kubernetes-novolume"} {
+		t.Run("containerMode "+mode, func(t *testing.T) {
+			helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
+			require.NoError(t, err)
+
+			releaseName := "test-runners"
+			namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+			options := &helm.Options{
+				Logger: logger.Discard,
+				SetValues: map[string]string{
+					"githubConfigUrl":                    "https://github.com/actions",
+					"githubConfigSecret.github_token":    "gh_token12345",
+					"controllerServiceAccount.name":      "arc",
+					"controllerServiceAccount.namespace": "arc-system",
+					"containerMode.type":                 mode,
+				},
+				KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+			}
+
+			_, err = helm.RenderTemplateE(
+				t,
+				options,
+				helmChartPath,
+				releaseName,
+				[]string{"templates/no_permission_serviceaccount.yaml"},
+			)
+			assert.ErrorContains(
+				t,
+				err,
+				"could not find template templates/no_permission_serviceaccount.yaml in chart",
+				"no permission service account should not be rendered in "+mode+" mode",
+			)
+		})
+	}
+}
+
 func TestTemplateRenderedUserProvideSetServiceAccount(t *testing.T) {
 	t.Parallel()
 

controllers/actions.github.com/autoscalinglistener_controller.go

@@ -692,7 +692,7 @@ func (r *AutoscalingListenerReconciler) publishRunningListener(autoscalingListen
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error {
 	labelBasedWatchFunc := func(_ context.Context, obj client.Object) []reconcile.Request {
 		var requests []reconcile.Request
 		labels := obj.GetLabels()
@@ -716,14 +716,16 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error
 		return requests
 	}
 
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&v1alpha1.AutoscalingListener{}).
-		Owns(&corev1.Pod{}).
-		Owns(&corev1.ServiceAccount{}).
-		Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
-		Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
-		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
-		Complete(r)
+	return builderWithOptions(
+		ctrl.NewControllerManagedBy(mgr).
+			For(&v1alpha1.AutoscalingListener{}).
+			Owns(&corev1.Pod{}).
+			Owns(&corev1.ServiceAccount{}).
+			Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
+			Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
+			WithEventFilter(predicate.ResourceVersionChangedPredicate{}),
+		opts,
+	).Complete(r)
 }
 
 func listenerContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus {

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -762,25 +762,27 @@ func (r *AutoscalingRunnerSetReconciler) listEphemeralRunnerSets(ctx context.Con
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&v1alpha1.AutoscalingRunnerSet{}).
-		Owns(&v1alpha1.EphemeralRunnerSet{}).
-		Watches(&v1alpha1.AutoscalingListener{}, handler.EnqueueRequestsFromMapFunc(
-			func(_ context.Context, o client.Object) []reconcile.Request {
-				autoscalingListener := o.(*v1alpha1.AutoscalingListener)
-				return []reconcile.Request{
-					{
-						NamespacedName: types.NamespacedName{
-							Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
-							Name:      autoscalingListener.Spec.AutoscalingRunnerSetName,
+func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error {
+	return builderWithOptions(
+		ctrl.NewControllerManagedBy(mgr).
+			For(&v1alpha1.AutoscalingRunnerSet{}).
+			Owns(&v1alpha1.EphemeralRunnerSet{}).
+			Watches(&v1alpha1.AutoscalingListener{}, handler.EnqueueRequestsFromMapFunc(
+				func(_ context.Context, o client.Object) []reconcile.Request {
+					autoscalingListener := o.(*v1alpha1.AutoscalingListener)
+					return []reconcile.Request{
+						{
+							NamespacedName: types.NamespacedName{
+								Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
+								Name:      autoscalingListener.Spec.AutoscalingRunnerSetName,
+							},
 						},
-					},
-				}
-			},
-		)).
-		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
-		Complete(r)
+					}
+				},
+			)).
+			WithEventFilter(predicate.ResourceVersionChangedPredicate{}),
+		opts,
+	).Complete(r)
 }
 
 type autoscalingRunnerSetFinalizerDependencyCleaner struct {

controllers/actions.github.com/ephemeralrunnerset_controller.go

@@ -522,12 +522,14 @@ func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ct
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&v1alpha1.EphemeralRunnerSet{}).
-		Owns(&v1alpha1.EphemeralRunner{}).
-		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
-		Complete(r)
+func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error {
+	return builderWithOptions(
+		ctrl.NewControllerManagedBy(mgr).
+			For(&v1alpha1.EphemeralRunnerSet{}).
+			Owns(&v1alpha1.EphemeralRunner{}).
+			WithEventFilter(predicate.ResourceVersionChangedPredicate{}),
+		opts,
+	).Complete(r)
 }
 
 type ephemeralRunnerStepper struct {

controllers/actions.github.com/options.go

@@ -1,8 +1,10 @@
 package actionsgithubcom
 
 import (
+	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 // Options is the optional configuration for the controllers, which can be
@@ -37,6 +39,25 @@ func WithMaxConcurrentReconciles(n int) Option {
 	}
 }
 
+// WithTypedRateLimiter sets the rate limiter for the controller's workqueue.
+//
+// By default, the controller-runtime uses
+// workqueue.DefaultTypedControllerRateLimiter[reconcile.Request], which combines
+// an exponential backoff per-item limiter with a token bucket overall limiter
+// (10 QPS, 100 bucket size). In large-scale environments with many runner
+// scale sets, the token bucket limiter can become a bottleneck for
+// reconciliation throughput.
+//
+// Use this option to override the default rate limiter, for example, to use
+// workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request], which removes
+// the overall token bucket constraint while keeping the per-item exponential
+// backoff.
+func WithTypedRateLimiter(rateLimiter workqueue.TypedRateLimiter[reconcile.Request]) Option {
+	return func(b *controller.Options) {
+		b.RateLimiter = rateLimiter
+	}
+}
+
 // builderWithOptions applies the given options to the provided builder, if any.
 // This is a helper function to avoid the need to import the controller-runtime package in every reconciler source file
 // and the command package that creates the controller.

main.go

@@ -39,10 +39,12 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	// +kubebuilder:scaffold:imports
 )
@@ -83,6 +85,7 @@ func main() {
 		listenerMetricsEndpoint string
 
 		metricsAddr              string
+		pprofAddr                string
 		autoScalingRunnerSetOnly bool
 		enableLeaderElection     bool
 		disableAdmissionWebhook  bool
@@ -110,6 +113,8 @@ func main() {
 
 		k8sClientRateLimiterQPS   int
 		k8sClientRateLimiterBurst int
+
+		workqueueRateLimiter string
 	)
 	var c github.Config
 	err = envconfig.Process("github", &c)
@@ -121,6 +126,7 @@ func main() {
 	flag.StringVar(&listenerMetricsAddr, "listener-metrics-addr", ":8080", "The address applied to AutoscalingListener metrics server")
 	flag.StringVar(&listenerMetricsEndpoint, "listener-metrics-endpoint", "/metrics", "The AutoscalingListener metrics server endpoint from which the metrics are collected")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&pprofAddr, "pprof-addr", "", "The address the pprof endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&leaderElectionID, "leader-election-id", "actions-runner-controller", "Controller id for leader election.")
@@ -155,6 +161,7 @@ func main() {
 	flag.Var(&autoScalerImagePullSecrets, "auto-scaler-image-pull-secrets", "The default image-pull secret name for auto-scaler listener container.")
 	flag.IntVar(&k8sClientRateLimiterQPS, "k8s-client-rate-limiter-qps", 20, "The QPS value of the K8s client rate limiter.")
 	flag.IntVar(&k8sClientRateLimiterBurst, "k8s-client-rate-limiter-burst", 30, "The burst value of the K8s client rate limiter.")
+	flag.StringVar(&workqueueRateLimiter, "workqueue-rate-limiter", "", `The workqueue rate limiter to use. Valid values are "bucket_rate_limiter" (default) and "typed_rate_limiter" (per-item only, no global token bucket).`)
 	flag.Parse()
 
 	runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets
@@ -239,6 +246,7 @@ func main() {
 			SyncPeriod:        &syncPeriod,
 			DefaultNamespaces: defaultNamespaces,
 		},
+		PprofBindAddress: pprofAddr,
 		WebhookServer:    webhookServer,
 		LeaderElection:   enableLeaderElection,
 		LeaderElectionID: leaderElectionID,
@@ -293,6 +301,20 @@ func main() {
 
 		log.Info("Resource builder initializing")
 
+		var controllerOpts []actionsgithubcom.Option
+		switch workqueueRateLimiter {
+		case "typed_rate_limiter":
+			log.Info("Using typed rate limiter (per-item only, no global token bucket)")
+			controllerOpts = append(controllerOpts,
+				actionsgithubcom.WithTypedRateLimiter(workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]()),
+			)
+		case "bucket_rate_limiter", "":
+			log.Info("Using default bucket rate limiter")
+		default:
+			log.Error(fmt.Errorf("unknown workqueue rate limiter: %s", workqueueRateLimiter), "invalid --workqueue-rate-limiter value")
+			os.Exit(1)
+		}
+
 		if err = (&actionsgithubcom.AutoscalingRunnerSetReconciler{
 			Client:                             mgr.GetClient(),
 			Log:                                log.WithName("AutoscalingRunnerSet").WithValues("version", build.Version),
@@ -302,17 +324,18 @@ func main() {
 			UpdateStrategy:                     actionsgithubcom.UpdateStrategy(updateStrategy),
 			DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets,
 			ResourceBuilder: rb,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, controllerOpts...); err != nil {
 			log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet")
 			os.Exit(1)
 		}
 
+		runnerOpts := append(controllerOpts, actionsgithubcom.WithMaxConcurrentReconciles(opts.RunnerMaxConcurrentReconciles))
 		if err = (&actionsgithubcom.EphemeralRunnerReconciler{
 			Client:          mgr.GetClient(),
 			Log:             log.WithName("EphemeralRunner").WithValues("version", build.Version),
 			Scheme:          mgr.GetScheme(),
 			ResourceBuilder: rb,
-		}).SetupWithManager(mgr, actionsgithubcom.WithMaxConcurrentReconciles(opts.RunnerMaxConcurrentReconciles)); err != nil {
+		}).SetupWithManager(mgr, runnerOpts...); err != nil {
 			log.Error(err, "unable to create controller", "controller", "EphemeralRunner")
 			os.Exit(1)
 		}
@@ -323,7 +346,7 @@ func main() {
 			Scheme:          mgr.GetScheme(),
 			PublishMetrics:  metricsAddr != "0",
 			ResourceBuilder: rb,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, controllerOpts...); err != nil {
 			log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet")
 			os.Exit(1)
 		}
@@ -335,7 +358,7 @@ func main() {
 			ListenerMetricsAddr:     listenerMetricsAddr,
 			ListenerMetricsEndpoint: listenerMetricsEndpoint,
 			ResourceBuilder:         rb,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, controllerOpts...); err != nil {
 			log.Error(err, "unable to create controller", "controller", "AutoscalingListener")
 			os.Exit(1)
 		}

runner/Makefile

@@ -6,7 +6,7 @@ DIND_ROOTLESS_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind-rootless
 OS_IMAGE ?= ubuntu-22.04
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.333.1
+RUNNER_VERSION ?= 2.334.0
 RUNNER_CONTAINER_HOOKS_VERSION ?= 0.8.1
 DOCKER_VERSION ?= 28.0.4
 

runner/VERSION

@@ -1,2 +1,2 @@
-RUNNER_VERSION=2.333.1
+RUNNER_VERSION=2.334.0
 RUNNER_CONTAINER_HOOKS_VERSION=0.8.1

test/e2e/e2e_test.go

@@ -36,7 +36,7 @@ var (
 
 	testResultCMNamePrefix = "test-result-"
 
-	RunnerVersion               = "2.333.1"
+	RunnerVersion               = "2.334.0"
 	RunnerContainerHooksVersion = "0.8.1"
 )