fix(rc-5): add Redis ACL persistence + cache reprovision endpoint
## Changes
### port.Deployer interface
- Add PatchProjectSecrets(ctx, projectName, patch) to merge key-value pairs
into all K8s secrets labeled project={projectName}
- Add RestartAll(ctx, projectName) to trigger rolling restart of all deployments
for a project, picking up fresh secrets without waiting for CI
### deployer adapter
- Implement PatchProjectSecrets: lists secrets by label, merges patch into Data,
writes each secret back
- Implement RestartAll: lists deployments by label, sets restartedAt annotation
### domain/credential.go
- Add CredentialCategoryCache = "cache" constant
- Use constant in component_infra.go (was raw string "cache")
### handlers/cache.go (new)
- POST /projects/{projectID}/cache/reprovision
- Calls CreateProjectCache (which handles delete+recreate with new password)
- Updates credential store (REDIS_URL, REDIS_URL_STAGING, REDIS_PREFIX)
- Patches all K8s secrets for the project immediately
- Triggers RestartAll so pods pick up new credentials without waiting for deploy
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
62a9bbb237
commit
17240f4efd
@ -639,6 +639,9 @@ func main() {
|
||||
// Initialize notify handler (domain status and re-verification)
|
||||
notifyHandler := handlers.NewNotifyHandler(notifyProvisioner, credentialStore, logger)
|
||||
|
||||
// Initialize cache handler (Redis reprovision after ACL reset)
|
||||
cacheHandler := handlers.NewCacheHandler(cacheProvisioner, credentialStore, deployerAdapter, logger)
|
||||
|
||||
// Initialize operations handler (for debugging project failures)
|
||||
operationsHandler := handlers.NewOperationsHandler(operationRepo)
|
||||
|
||||
@ -722,6 +725,7 @@ func main() {
|
||||
verifyHandler.Mount(app.Router())
|
||||
sagaHandler.Mount(app.Router())
|
||||
notifyHandler.Mount(app.Router())
|
||||
cacheHandler.Mount(app.Router())
|
||||
|
||||
// Start queue processor worker (per-project command queue)
|
||||
queueProcessor := worker.NewQueueProcessor(
|
||||
|
||||
@ -184,6 +184,56 @@ func (d *Deployer) ListComponentStatuses(ctx context.Context, projectName string
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PatchProjectSecrets merges key-value pairs into all K8s secrets labeled project={projectName}.
|
||||
// Existing keys not present in patch are preserved.
|
||||
func (d *Deployer) PatchProjectSecrets(ctx context.Context, projectName string, patch map[string]string) error {
|
||||
ns := d.config.Namespace
|
||||
secretList, err := d.client.CoreV1().Secrets(ns).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("project=%s", projectName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list secrets for project %s: %w", projectName, err)
|
||||
}
|
||||
|
||||
for i := range secretList.Items {
|
||||
s := &secretList.Items[i]
|
||||
if s.Data == nil {
|
||||
s.Data = make(map[string][]byte)
|
||||
}
|
||||
for k, v := range patch {
|
||||
s.Data[k] = []byte(v)
|
||||
}
|
||||
if _, err := d.client.CoreV1().Secrets(ns).Update(ctx, s, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("update secret %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestartAll triggers a rolling restart of all deployments labeled project={projectName}.
|
||||
func (d *Deployer) RestartAll(ctx context.Context, projectName string) error {
|
||||
ns := d.config.Namespace
|
||||
deploymentList, err := d.client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("project=%s", projectName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list deployments for project %s: %w", projectName, err)
|
||||
}
|
||||
|
||||
restartedAt := time.Now().Format(time.RFC3339)
|
||||
for i := range deploymentList.Items {
|
||||
dep := &deploymentList.Items[i]
|
||||
if dep.Spec.Template.Annotations == nil {
|
||||
dep.Spec.Template.Annotations = make(map[string]string)
|
||||
}
|
||||
dep.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = restartedAt
|
||||
if _, err := d.client.AppsV1().Deployments(ns).Update(ctx, dep, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("restart deployment %s: %w", dep.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitComponentPath splits a component path like "services/auth-api" into ["services", "auth-api"].
|
||||
func splitComponentPath(path string) []string {
|
||||
var parts []string
|
||||
|
||||
@ -39,6 +39,7 @@ const (
|
||||
CredentialCategoryStorage = "storage"
|
||||
CredentialCategoryAI = "ai"
|
||||
CredentialCategoryNotify = "notify"
|
||||
CredentialCategoryCache = "cache"
|
||||
)
|
||||
|
||||
// Known credential keys.
|
||||
|
||||
134
internal/handlers/cache.go
Normal file
134
internal/handlers/cache.go
Normal file
@ -0,0 +1,134 @@
|
||||
// Package handlers provides HTTP handlers for the rdev API.
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/orchard9/rdev/internal/auth"
|
||||
"github.com/orchard9/rdev/internal/domain"
|
||||
"github.com/orchard9/rdev/internal/logging"
|
||||
"github.com/orchard9/rdev/internal/port"
|
||||
"github.com/orchard9/rdev/pkg/api"
|
||||
)
|
||||
|
||||
// CacheHandler handles cache reprovision endpoints.
|
||||
type CacheHandler struct {
|
||||
cacheProvisioner port.CacheProvisioner // may be nil if not configured
|
||||
credStore port.CredentialStore // may be nil
|
||||
deployer port.Deployer // may be nil
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewCacheHandler creates a new cache handler.
|
||||
func NewCacheHandler(
|
||||
cacheProvisioner port.CacheProvisioner,
|
||||
credStore port.CredentialStore,
|
||||
deployer port.Deployer,
|
||||
logger *slog.Logger,
|
||||
) *CacheHandler {
|
||||
return &CacheHandler{
|
||||
cacheProvisioner: cacheProvisioner,
|
||||
credStore: credStore,
|
||||
deployer: deployer,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// Mount registers the cache routes.
|
||||
func (h *CacheHandler) Mount(r api.Router) {
|
||||
r.Route("/projects/{projectID}/cache", func(r chi.Router) {
|
||||
r.With(auth.RequireScope(auth.ScopeProjectsExecute, auth.ScopeAdmin)).
|
||||
Post("/reprovision", h.Reprovision)
|
||||
})
|
||||
}
|
||||
|
||||
// Reprovision recreates the Redis ACL user with a new password, updates the credential
|
||||
// store, patches all K8s secrets for the project, and triggers rolling restarts so
|
||||
// pods pick up the new credentials immediately.
|
||||
// POST /projects/{projectID}/cache/reprovision
|
||||
func (h *CacheHandler) Reprovision(w http.ResponseWriter, r *http.Request) {
|
||||
if h.cacheProvisioner == nil {
|
||||
api.WriteError(w, r, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "cache provisioner not configured")
|
||||
return
|
||||
}
|
||||
|
||||
projectID := chi.URLParam(r, "projectID")
|
||||
if projectID == "" {
|
||||
api.WriteBadRequest(w, r, "project ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), TimeoutStandard)
|
||||
defer cancel()
|
||||
|
||||
log := logging.FromContext(ctx).WithHandler("CacheReprovision")
|
||||
|
||||
// CreateProjectCache handles delete+recreate: if the ACL user already exists it
|
||||
// is deleted first, then a new one is created with a fresh password.
|
||||
creds, err := h.cacheProvisioner.CreateProjectCache(ctx, projectID)
|
||||
if err != nil {
|
||||
log.Error("failed to reprovision cache",
|
||||
logging.FieldError, err,
|
||||
logging.FieldProjectID, projectID,
|
||||
)
|
||||
api.WriteInternalError(w, r, "failed to reprovision cache")
|
||||
return
|
||||
}
|
||||
|
||||
// Persist new credentials in the credential store.
|
||||
if h.credStore != nil {
|
||||
for _, kv := range []struct{ key, val string }{
|
||||
{"REDIS_URL", creds.URL},
|
||||
{"REDIS_URL_STAGING", creds.URLStaging},
|
||||
{"REDIS_PREFIX", creds.Prefix},
|
||||
} {
|
||||
if err := h.credStore.Set(ctx, domain.Credential{
|
||||
Key: projectID + ":" + kv.key,
|
||||
Value: kv.val,
|
||||
Category: domain.CredentialCategoryCache,
|
||||
}); err != nil {
|
||||
log.Error("failed to store cache credential",
|
||||
logging.FieldError, err,
|
||||
logging.FieldProjectID, projectID,
|
||||
"credential_key", kv.key,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update K8s secrets and restart all project deployments so pods pick up the
|
||||
// new credentials without waiting for the next CI deploy.
|
||||
if h.deployer != nil {
|
||||
patch := map[string]string{
|
||||
"REDIS_URL": creds.URL,
|
||||
"REDIS_URL_STAGING": creds.URLStaging,
|
||||
"REDIS_PREFIX": creds.Prefix,
|
||||
}
|
||||
if err := h.deployer.PatchProjectSecrets(ctx, projectID, patch); err != nil {
|
||||
log.Warn("failed to patch K8s secrets; pods will get new creds on next CI deploy",
|
||||
logging.FieldError, err,
|
||||
logging.FieldProjectID, projectID,
|
||||
)
|
||||
} else if err := h.deployer.RestartAll(ctx, projectID); err != nil {
|
||||
log.Warn("failed to restart project deployments",
|
||||
logging.FieldError, err,
|
||||
logging.FieldProjectID, projectID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("cache reprovisioned",
|
||||
logging.FieldProjectID, projectID,
|
||||
"prefix", creds.Prefix,
|
||||
"username", creds.Username,
|
||||
)
|
||||
|
||||
api.WriteSuccess(w, r, map[string]string{
|
||||
"prefix": creds.Prefix,
|
||||
"username": creds.Username,
|
||||
"status": "reprovisioned",
|
||||
})
|
||||
}
|
||||
@ -342,6 +342,14 @@ func (m *mockDeployer) RemoveIngressPath(_ context.Context, _, _, _ string) erro
|
||||
return m.err
|
||||
}
|
||||
|
||||
func (m *mockDeployer) PatchProjectSecrets(_ context.Context, _ string, _ map[string]string) error {
|
||||
return m.err
|
||||
}
|
||||
|
||||
func (m *mockDeployer) RestartAll(_ context.Context, _ string) error {
|
||||
return m.err
|
||||
}
|
||||
|
||||
// mockPreviewManager implements port.PreviewManager for testing.
|
||||
type mockPreviewManager struct {
|
||||
previews map[string]bool
|
||||
|
||||
@ -74,4 +74,13 @@ type Deployer interface {
|
||||
// If no paths remain for a host, the host rule is removed.
|
||||
// If no rules remain, the Ingress is deleted.
|
||||
RemoveIngressPath(ctx context.Context, projectName, host, path string) error
|
||||
|
||||
// PatchProjectSecrets merges the given key-value pairs into all K8s secrets
|
||||
// belonging to the project (labeled project={projectName}). Existing keys not
|
||||
// present in patch are left untouched.
|
||||
PatchProjectSecrets(ctx context.Context, projectName string, patch map[string]string) error
|
||||
|
||||
// RestartAll triggers a rolling restart of all deployments belonging to the project
|
||||
// (labeled project={projectName}). Used after credential rotation to pick up new secrets.
|
||||
RestartAll(ctx context.Context, projectName string) error
|
||||
}
|
||||
|
||||
@ -113,7 +113,7 @@ func (s *ComponentService) provisionRedis(ctx context.Context, projectID, name s
|
||||
// Store credentials if credential store is available
|
||||
log := logging.FromContext(ctx).WithService("component")
|
||||
if s.credentialStore != nil {
|
||||
if err := s.storeCredential(ctx, projectID, "cache", "REDIS_URL", creds.URL); err != nil {
|
||||
if err := s.storeCredential(ctx, projectID, domain.CredentialCategoryCache, "REDIS_URL", creds.URL); err != nil {
|
||||
// Rollback on credential storage failure
|
||||
log.Error("failed to store REDIS_URL, rolling back", logging.FieldError, err)
|
||||
if rollbackErr := s.cacheProvisioner.DeleteProjectCache(ctx, projectID, false); rollbackErr != nil {
|
||||
@ -121,10 +121,10 @@ func (s *ComponentService) provisionRedis(ctx context.Context, projectID, name s
|
||||
}
|
||||
return nil, fmt.Errorf("failed to store credentials: %w", err)
|
||||
}
|
||||
if err := s.storeCredential(ctx, projectID, "cache", "REDIS_URL_STAGING", creds.URLStaging); err != nil {
|
||||
if err := s.storeCredential(ctx, projectID, domain.CredentialCategoryCache, "REDIS_URL_STAGING", creds.URLStaging); err != nil {
|
||||
log.Warn("failed to store REDIS_URL_STAGING", logging.FieldError, err)
|
||||
}
|
||||
if err := s.storeCredential(ctx, projectID, "cache", "REDIS_PREFIX", creds.Prefix); err != nil {
|
||||
if err := s.storeCredential(ctx, projectID, domain.CredentialCategoryCache, "REDIS_PREFIX", creds.Prefix); err != nil {
|
||||
log.Warn("failed to store REDIS_PREFIX", logging.FieldError, err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user