feat: Complete automation gaps for repeatable project deployments
- Initial K8s deployment auto-creation during project creation - DNS record upsert support (create or update existing records) - Ingress host management for domain aliases (AddIngressHost/RemoveIngressHost) - Woodpecker deployer RBAC manifest for CI deploy steps - Single-commit template seeding via Gitea bulk file API Closes automation gaps exposed during www.threesix.ai launch: - Projects now auto-create K8s Deployment/Service/Ingress on creation - Domain aliases automatically update both DNS and K8s ingress - CI deploy steps work without manual RBAC setup - Template seeding triggers only one CI pipeline (not per-file) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
79b32ffa6c
commit
34e72687e6
@ -215,11 +215,11 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize template provider (requires Gitea client for seeding repos)
|
// Initialize template provider (requires Gitea credentials for seeding repos)
|
||||||
var templateProvider *templates.Provider
|
var templateProvider *templates.Provider
|
||||||
if giteaClient != nil {
|
if infraCfg.GiteaToken != "" && infraCfg.GiteaURL != "" {
|
||||||
// Get the underlying Gitea SDK client for the template provider
|
// Pass URL and token directly - provider uses bulk file API for single-commit seeding
|
||||||
templateProvider = templates.NewProvider(giteaClient.SDKClient(), logger)
|
templateProvider = templates.NewProvider(infraCfg.GiteaURL, infraCfg.GiteaToken, logger)
|
||||||
logger.Info("template provider initialized")
|
logger.Info("template provider initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -58,7 +58,7 @@ All infrastructure gaps have been closed. The full pipeline from project creatio
|
|||||||
- [x] rdev-api running with infrastructure handlers
|
- [x] rdev-api running with infrastructure handlers
|
||||||
- [x] Gitea at https://git.threesix.ai
|
- [x] Gitea at https://git.threesix.ai
|
||||||
- [x] Woodpecker CI at https://ci.threesix.ai
|
- [x] Woodpecker CI at https://ci.threesix.ai
|
||||||
- [x] Zot registry at zot.threesix.svc.cluster.local:5000
|
- [x] Zot registry at registry.threesix.ai
|
||||||
- [x] `projects` namespace in K8s with RBAC
|
- [x] `projects` namespace in K8s with RBAC
|
||||||
- [x] Wildcard TLS cert for *.threesix.ai
|
- [x] Wildcard TLS cert for *.threesix.ai
|
||||||
|
|
||||||
|
|||||||
@ -27,6 +27,9 @@ resources:
|
|||||||
# v0.4+ - API Server (RBAC now included in rdev-api.yaml)
|
# v0.4+ - API Server (RBAC now included in rdev-api.yaml)
|
||||||
- rdev-api.yaml
|
- rdev-api.yaml
|
||||||
|
|
||||||
|
# Woodpecker CI RBAC - allows deploy steps to update deployments in projects namespace
|
||||||
|
- woodpecker-deployer-rbac.yaml
|
||||||
|
|
||||||
# v0.8+ - Production hardening
|
# v0.8+ - Production hardening
|
||||||
- pdb.yaml
|
- pdb.yaml
|
||||||
- network-policy.yaml
|
- network-policy.yaml
|
||||||
|
|||||||
49
deployments/k8s/base/woodpecker-deployer-rbac.yaml
Normal file
49
deployments/k8s/base/woodpecker-deployer-rbac.yaml
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
# RBAC for Woodpecker CI to deploy projects
|
||||||
|
#
|
||||||
|
# The Woodpecker CI deploy step runs as the `default` ServiceAccount in the
|
||||||
|
# `threesix` namespace but needs to update deployments in the `projects`
|
||||||
|
# namespace using `kubectl set image`.
|
||||||
|
#
|
||||||
|
# This uses a namespace-scoped Role (not ClusterRole) to follow least-privilege:
|
||||||
|
# permissions are restricted to the `projects` namespace only.
|
||||||
|
#
|
||||||
|
# Without this, deploy steps fail with:
|
||||||
|
# Error from server (Forbidden): deployments.apps "project-name" is forbidden:
|
||||||
|
# User "system:serviceaccount:threesix:default" cannot patch resource
|
||||||
|
# "deployments" in API group "apps" in the namespace "projects"
|
||||||
|
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: woodpecker-deployer
|
||||||
|
namespace: projects # Scoped to projects namespace only
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: woodpecker-deployer
|
||||||
|
app.kubernetes.io/part-of: rdev
|
||||||
|
rules:
|
||||||
|
# Minimal permissions for `kubectl set image` on deployments
|
||||||
|
# - get: Required to read current deployment state
|
||||||
|
# - list: Required for kubectl to find the deployment
|
||||||
|
# - patch: Required for `kubectl set image` to update the container image
|
||||||
|
- apiGroups: ["apps"]
|
||||||
|
resources: ["deployments"]
|
||||||
|
verbs: ["get", "list", "patch"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: woodpecker-deployer
|
||||||
|
namespace: projects # Binding in the target namespace
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: woodpecker-deployer
|
||||||
|
app.kubernetes.io/part-of: rdev
|
||||||
|
subjects:
|
||||||
|
# Woodpecker CI runs pipeline steps as the default ServiceAccount
|
||||||
|
# in the threesix namespace
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: default
|
||||||
|
namespace: threesix
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: woodpecker-deployer
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/orchard9/rdev/internal/domain"
|
"github.com/orchard9/rdev/internal/domain"
|
||||||
@ -195,6 +196,58 @@ func (c *Client) FindRecord(ctx context.Context, recordType, name string) (*doma
|
|||||||
return recordFromCFMap(result.Result[0]), nil
|
return recordFromCFMap(result.Result[0]), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpsertRecord creates or updates a DNS record.
|
||||||
|
// If a record with the same type and name exists, it updates it.
|
||||||
|
// Otherwise, it creates a new record.
|
||||||
|
// Returns the created or updated record.
|
||||||
|
// Handles race conditions with retry logic.
|
||||||
|
func (c *Client) UpsertRecord(ctx context.Context, record domain.DNSRecord) (*domain.DNSRecord, error) {
|
||||||
|
const maxRetries = 3
|
||||||
|
|
||||||
|
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||||
|
// Try to find existing record
|
||||||
|
existing, err := c.FindRecord(ctx, record.Type, record.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to check for existing record: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if existing != nil {
|
||||||
|
// Update existing record
|
||||||
|
updated, err := c.UpdateRecord(ctx, existing.ID, record)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to update existing record: %w", err)
|
||||||
|
}
|
||||||
|
return updated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create new record
|
||||||
|
created, err := c.CreateRecord(ctx, record)
|
||||||
|
if err == nil {
|
||||||
|
return created, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle race condition: record was created between Find and Create
|
||||||
|
if !isRecordExistsError(err) {
|
||||||
|
return nil, fmt.Errorf("failed to create record: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Race condition detected - retry the whole find-or-create loop
|
||||||
|
// This handles the case where another process created the record
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("failed to upsert record after %d attempts due to concurrent modifications", maxRetries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRecordExistsError checks if the error indicates a duplicate record.
|
||||||
|
func isRecordExistsError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Cloudflare returns "A record with that host already exists" or similar
|
||||||
|
errStr := err.Error()
|
||||||
|
return strings.Contains(errStr, "already exists") || strings.Contains(errStr, "duplicate")
|
||||||
|
}
|
||||||
|
|
||||||
// normalizeName converts a subdomain to full domain name.
|
// normalizeName converts a subdomain to full domain name.
|
||||||
func (c *Client) normalizeName(name string) string {
|
func (c *Client) normalizeName(name string) string {
|
||||||
if name == "@" || name == "" {
|
if name == "@" || name == "" {
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package deployer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
appsv1 "k8s.io/api/apps/v1"
|
appsv1 "k8s.io/api/apps/v1"
|
||||||
@ -266,3 +267,114 @@ func resourceQuantity(s string) resource.Quantity {
|
|||||||
q, _ := resource.ParseQuantity(s)
|
q, _ := resource.ParseQuantity(s)
|
||||||
return q
|
return q
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddIngressHost adds a new host to an existing project's ingress.
|
||||||
|
// This is used when adding domain aliases to a project.
|
||||||
|
// The host is added to both the TLS configuration and the routing rules.
|
||||||
|
func (d *Deployer) AddIngressHost(ctx context.Context, projectName, host string) error {
|
||||||
|
ns := d.config.Namespace
|
||||||
|
|
||||||
|
// Get existing ingress
|
||||||
|
ingress, err := d.client.NetworkingV1().Ingresses(ns).Get(ctx, projectName, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get ingress: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if host already exists
|
||||||
|
for _, rule := range ingress.Spec.Rules {
|
||||||
|
if rule.Host == host {
|
||||||
|
return nil // Host already exists, nothing to do
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the service port from existing rules (assumes all rules use same backend)
|
||||||
|
var servicePort int32 = 80
|
||||||
|
if len(ingress.Spec.Rules) > 0 && ingress.Spec.Rules[0].HTTP != nil && len(ingress.Spec.Rules[0].HTTP.Paths) > 0 {
|
||||||
|
servicePort = ingress.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Port.Number
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add TLS entry for the new host
|
||||||
|
tlsSecretName := strings.ReplaceAll(host, ".", "-") + "-tls"
|
||||||
|
ingress.Spec.TLS = append(ingress.Spec.TLS, networkingv1.IngressTLS{
|
||||||
|
Hosts: []string{host},
|
||||||
|
SecretName: tlsSecretName,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Add routing rule for the new host
|
||||||
|
pathType := networkingv1.PathTypePrefix
|
||||||
|
ingress.Spec.Rules = append(ingress.Spec.Rules, networkingv1.IngressRule{
|
||||||
|
Host: host,
|
||||||
|
IngressRuleValue: networkingv1.IngressRuleValue{
|
||||||
|
HTTP: &networkingv1.HTTPIngressRuleValue{
|
||||||
|
Paths: []networkingv1.HTTPIngressPath{
|
||||||
|
{
|
||||||
|
Path: "/",
|
||||||
|
PathType: &pathType,
|
||||||
|
Backend: networkingv1.IngressBackend{
|
||||||
|
Service: &networkingv1.IngressServiceBackend{
|
||||||
|
Name: projectName,
|
||||||
|
Port: networkingv1.ServiceBackendPort{
|
||||||
|
Number: servicePort,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Update the ingress
|
||||||
|
_, err = d.client.NetworkingV1().Ingresses(ns).Update(ctx, ingress, metav1.UpdateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to update ingress: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveIngressHost removes a host from an existing project's ingress.
|
||||||
|
// This is used when removing domain aliases from a project.
|
||||||
|
func (d *Deployer) RemoveIngressHost(ctx context.Context, projectName, host string) error {
|
||||||
|
ns := d.config.Namespace
|
||||||
|
|
||||||
|
// Get existing ingress
|
||||||
|
ingress, err := d.client.NetworkingV1().Ingresses(ns).Get(ctx, projectName, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get ingress: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove from TLS entries
|
||||||
|
var newTLS []networkingv1.IngressTLS
|
||||||
|
for _, tls := range ingress.Spec.TLS {
|
||||||
|
// Keep TLS entries that don't contain this host
|
||||||
|
var newHosts []string
|
||||||
|
for _, h := range tls.Hosts {
|
||||||
|
if h != host {
|
||||||
|
newHosts = append(newHosts, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(newHosts) > 0 {
|
||||||
|
tls.Hosts = newHosts
|
||||||
|
newTLS = append(newTLS, tls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ingress.Spec.TLS = newTLS
|
||||||
|
|
||||||
|
// Remove from routing rules
|
||||||
|
var newRules []networkingv1.IngressRule
|
||||||
|
for _, rule := range ingress.Spec.Rules {
|
||||||
|
if rule.Host != host {
|
||||||
|
newRules = append(newRules, rule)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ingress.Spec.Rules = newRules
|
||||||
|
|
||||||
|
// Update the ingress
|
||||||
|
_, err = d.client.NetworkingV1().Ingresses(ns).Update(ctx, ingress, metav1.UpdateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to update ingress: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
185
internal/adapter/gitea/bulk_files.go
Normal file
185
internal/adapter/gitea/bulk_files.go
Normal file
@ -0,0 +1,185 @@
|
|||||||
|
// Package gitea provides a Gitea API adapter implementing port.GitRepository.
|
||||||
|
package gitea
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ChangeFileOperation represents a single file operation in a bulk change.
|
||||||
|
type ChangeFileOperation struct {
|
||||||
|
// Operation is "create", "update", or "delete"
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
// Path is the file path (max 500 characters)
|
||||||
|
Path string `json:"path"`
|
||||||
|
// Content is the base64-encoded file content (required for create/update)
|
||||||
|
Content string `json:"content,omitempty"`
|
||||||
|
// SHA is required for update and delete operations
|
||||||
|
SHA string `json:"sha,omitempty"`
|
||||||
|
// FromPath is the original path when moving/renaming files
|
||||||
|
FromPath string `json:"from_path,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChangeFilesOptions options for changing multiple files in a single commit.
|
||||||
|
type ChangeFilesOptions struct {
|
||||||
|
// Files is the list of file operations
|
||||||
|
Files []ChangeFileOperation `json:"files"`
|
||||||
|
// Message is the commit message
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
|
// BranchName is the branch to commit to (optional, defaults to repo default)
|
||||||
|
BranchName string `json:"branch,omitempty"`
|
||||||
|
// NewBranchName creates a new branch (optional)
|
||||||
|
NewBranchName string `json:"new_branch,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FilesResponse is the response from bulk file operations.
|
||||||
|
type FilesResponse struct {
|
||||||
|
Files []FileContentResponse `json:"files"`
|
||||||
|
Commit *FileCommitResponse `json:"commit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileContentResponse represents a file in the response.
|
||||||
|
type FileContentResponse struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA string `json:"sha"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
HTMLURL string `json:"html_url"`
|
||||||
|
GitURL string `json:"git_url"`
|
||||||
|
DownloadURL string `json:"download_url"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileCommitResponse contains commit information from the response.
|
||||||
|
type FileCommitResponse struct {
|
||||||
|
SHA string `json:"sha"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
HTMLURL string `json:"html_url"`
|
||||||
|
Author *User `json:"author"`
|
||||||
|
Committer *User `json:"committer"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// User represents a git user in commit info.
|
||||||
|
type User struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Email string `json:"email"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BulkFileClient wraps a Gitea client with direct HTTP capabilities
|
||||||
|
// for operations not yet supported by the SDK.
|
||||||
|
type BulkFileClient struct {
|
||||||
|
baseURL string
|
||||||
|
token string
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBulkFileClient creates a new client for bulk file operations.
|
||||||
|
// baseURL is the Gitea server URL (e.g., https://git.threesix.ai)
|
||||||
|
// token is an API access token with repo permissions
|
||||||
|
func NewBulkFileClient(baseURL, token string) *BulkFileClient {
|
||||||
|
return &BulkFileClient{
|
||||||
|
baseURL: strings.TrimSuffix(baseURL, "/"),
|
||||||
|
token: token,
|
||||||
|
client: &http.Client{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChangeFiles creates, updates, or deletes multiple files in a single commit.
|
||||||
|
// This uses the Gitea API endpoint POST /repos/{owner}/{repo}/contents
|
||||||
|
// which was added in Gitea v1.20.0 (PR #24887).
|
||||||
|
// Includes retry with exponential backoff for transient failures.
|
||||||
|
func (c *BulkFileClient) ChangeFiles(ctx context.Context, owner, repo string, opts ChangeFilesOptions) (*FilesResponse, error) {
|
||||||
|
const maxRetries = 3
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||||
|
result, err := c.doChangeFiles(ctx, owner, repo, opts)
|
||||||
|
if err == nil {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
|
||||||
|
// Don't retry client errors (4xx) except rate limiting (429)
|
||||||
|
if !isRetryableError(err) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait before retry with exponential backoff
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(time.Duration(attempt+1) * time.Second):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// doChangeFiles performs the actual API request.
|
||||||
|
func (c *BulkFileClient) doChangeFiles(ctx context.Context, owner, repo string, opts ChangeFilesOptions) (*FilesResponse, error) {
|
||||||
|
url := fmt.Sprintf("%s/api/v1/repos/%s/%s/contents", c.baseURL, owner, repo)
|
||||||
|
|
||||||
|
body, err := json.Marshal(&opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "token "+c.token)
|
||||||
|
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("request failed: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
respBody, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return nil, &apiError{StatusCode: resp.StatusCode, Body: string(respBody)}
|
||||||
|
}
|
||||||
|
|
||||||
|
var result FilesResponse
|
||||||
|
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// apiError represents an API error with status code.
|
||||||
|
type apiError struct {
|
||||||
|
StatusCode int
|
||||||
|
Body string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *apiError) Error() string {
|
||||||
|
return fmt.Sprintf("API error (status %d): %s", e.StatusCode, e.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRetryableError checks if an error should be retried.
|
||||||
|
func isRetryableError(err error) bool {
|
||||||
|
if apiErr, ok := err.(*apiError); ok {
|
||||||
|
// Retry server errors (5xx) and rate limiting (429)
|
||||||
|
return apiErr.StatusCode >= 500 || apiErr.StatusCode == 429
|
||||||
|
}
|
||||||
|
// Retry network errors
|
||||||
|
return strings.Contains(err.Error(), "connection") ||
|
||||||
|
strings.Contains(err.Error(), "timeout") ||
|
||||||
|
strings.Contains(err.Error(), "EOF")
|
||||||
|
}
|
||||||
@ -3,6 +3,11 @@
|
|||||||
// Templates are embedded at compile time from the templates/ subdirectory.
|
// Templates are embedded at compile time from the templates/ subdirectory.
|
||||||
// Each template contains starter files with {{VAR}} placeholders that get
|
// Each template contains starter files with {{VAR}} placeholders that get
|
||||||
// interpolated when seeding a repository.
|
// interpolated when seeding a repository.
|
||||||
|
//
|
||||||
|
// Single-Commit Seeding:
|
||||||
|
// Template files are created in a single commit using Gitea's bulk file API
|
||||||
|
// (POST /repos/{owner}/{repo}/contents). This prevents multiple CI pipeline
|
||||||
|
// triggers that would occur with per-file commits.
|
||||||
package templates
|
package templates
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -16,7 +21,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/sdk/gitea"
|
giteaadapter "github.com/orchard9/rdev/internal/adapter/gitea"
|
||||||
"github.com/orchard9/rdev/internal/domain"
|
"github.com/orchard9/rdev/internal/domain"
|
||||||
"github.com/orchard9/rdev/internal/port"
|
"github.com/orchard9/rdev/internal/port"
|
||||||
)
|
)
|
||||||
@ -37,7 +42,7 @@ var templateNameRegex = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
|
|||||||
// Provider implements port.TemplateProvider using embedded templates
|
// Provider implements port.TemplateProvider using embedded templates
|
||||||
// and the Gitea API to seed repositories.
|
// and the Gitea API to seed repositories.
|
||||||
type Provider struct {
|
type Provider struct {
|
||||||
giteaClient *gitea.Client
|
bulkClient *giteaadapter.BulkFileClient
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -45,19 +50,22 @@ type Provider struct {
|
|||||||
var _ port.TemplateProvider = (*Provider)(nil)
|
var _ port.TemplateProvider = (*Provider)(nil)
|
||||||
|
|
||||||
// NewProvider creates a new template provider.
|
// NewProvider creates a new template provider.
|
||||||
// giteaClient is used to create files in repositories.
|
// giteaURL is the Gitea server URL (e.g., https://git.threesix.ai)
|
||||||
|
// giteaToken is an API access token with repo permissions
|
||||||
// logger is optional; if nil, slog.Default() is used.
|
// logger is optional; if nil, slog.Default() is used.
|
||||||
func NewProvider(giteaClient *gitea.Client, logger *slog.Logger) *Provider {
|
func NewProvider(giteaURL, giteaToken string, logger *slog.Logger) *Provider {
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
logger = slog.Default()
|
logger = slog.Default()
|
||||||
}
|
}
|
||||||
return &Provider{
|
return &Provider{
|
||||||
giteaClient: giteaClient,
|
bulkClient: giteaadapter.NewBulkFileClient(giteaURL, giteaToken),
|
||||||
logger: logger,
|
logger: logger,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SeedRepo populates a repository with template files.
|
// SeedRepo populates a repository with template files in a single commit.
|
||||||
|
// All template files are collected and committed atomically using Gitea's
|
||||||
|
// bulk file API, preventing multiple CI pipeline triggers.
|
||||||
func (p *Provider) SeedRepo(ctx context.Context, owner, repo, templateName string, vars map[string]string) error {
|
func (p *Provider) SeedRepo(ctx context.Context, owner, repo, templateName string, vars map[string]string) error {
|
||||||
// Check for context cancellation
|
// Check for context cancellation
|
||||||
select {
|
select {
|
||||||
@ -83,8 +91,8 @@ func (p *Provider) SeedRepo(ctx context.Context, owner, repo, templateName strin
|
|||||||
"template", templateName,
|
"template", templateName,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Walk template directory and create files
|
// Collect all template files
|
||||||
var filesCreated int
|
var fileOps []giteaadapter.ChangeFileOperation
|
||||||
err := fs.WalkDir(templatesFS, templateDir, func(path string, d fs.DirEntry, err error) error {
|
err := fs.WalkDir(templatesFS, templateDir, func(path string, d fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -111,35 +119,34 @@ func (p *Provider) SeedRepo(ctx context.Context, owner, repo, templateName strin
|
|||||||
// Strip .tmpl extension (allows embedding go.mod as go.mod.tmpl)
|
// Strip .tmpl extension (allows embedding go.mod as go.mod.tmpl)
|
||||||
relPath = strings.TrimSuffix(relPath, ".tmpl")
|
relPath = strings.TrimSuffix(relPath, ".tmpl")
|
||||||
|
|
||||||
// Create file in repo via Gitea API
|
|
||||||
// Gitea expects base64-encoded content
|
// Gitea expects base64-encoded content
|
||||||
encodedContent := base64.StdEncoding.EncodeToString([]byte(interpolated))
|
encodedContent := base64.StdEncoding.EncodeToString([]byte(interpolated))
|
||||||
|
|
||||||
// For empty repos (AutoInit: false), the first file must create the branch
|
fileOps = append(fileOps, giteaadapter.ChangeFileOperation{
|
||||||
// using NewBranchName. Subsequent files use the existing branch.
|
Operation: "create",
|
||||||
opts := gitea.CreateFileOptions{
|
Path: relPath,
|
||||||
Content: encodedContent,
|
Content: encodedContent,
|
||||||
FileOptions: gitea.FileOptions{
|
})
|
||||||
Message: "Add " + relPath + " from template",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if filesCreated == 0 {
|
|
||||||
// First file: create the main branch
|
|
||||||
opts.NewBranchName = "main"
|
|
||||||
} else {
|
|
||||||
// Subsequent files: use existing main branch
|
|
||||||
opts.BranchName = "main"
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _, err = p.giteaClient.CreateFile(owner, repo, relPath, opts)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create file %s: %w", relPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
filesCreated++
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to collect template files: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fileOps) == 0 {
|
||||||
|
return fmt.Errorf("template %s contains no files", templateName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create all files in a single commit
|
||||||
|
opts := giteaadapter.ChangeFilesOptions{
|
||||||
|
Files: fileOps,
|
||||||
|
Message: fmt.Sprintf("Initialize project from %s template", templateName),
|
||||||
|
NewBranchName: "main", // Create the main branch (repo is empty)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = p.bulkClient.ChangeFiles(ctx, owner, repo, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to seed repo from template %s: %w", templateName, err)
|
return fmt.Errorf("failed to seed repo from template %s: %w", templateName, err)
|
||||||
}
|
}
|
||||||
@ -148,7 +155,8 @@ func (p *Provider) SeedRepo(ctx context.Context, owner, repo, templateName strin
|
|||||||
"owner", owner,
|
"owner", owner,
|
||||||
"repo", repo,
|
"repo", repo,
|
||||||
"template", templateName,
|
"template", templateName,
|
||||||
"files_created", filesCreated,
|
"files_created", len(fileOps),
|
||||||
|
"commit_message", opts.Message,
|
||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@ -173,6 +173,18 @@ func (m *mockDNSProvider) FindRecord(_ context.Context, _, name string) (*domain
|
|||||||
return r, nil
|
return r, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockDNSProvider) UpsertRecord(ctx context.Context, record domain.DNSRecord) (*domain.DNSRecord, error) {
|
||||||
|
if m.err != nil {
|
||||||
|
return nil, m.err
|
||||||
|
}
|
||||||
|
// Check if record exists, then update or create
|
||||||
|
existing, _ := m.FindRecord(ctx, record.Type, record.Name)
|
||||||
|
if existing != nil {
|
||||||
|
return m.UpdateRecord(ctx, existing.ID, record)
|
||||||
|
}
|
||||||
|
return m.CreateRecord(ctx, record)
|
||||||
|
}
|
||||||
|
|
||||||
// mockDeployer implements port.Deployer for testing.
|
// mockDeployer implements port.Deployer for testing.
|
||||||
type mockDeployer struct {
|
type mockDeployer struct {
|
||||||
deployments map[string]*domain.DeployStatus
|
deployments map[string]*domain.DeployStatus
|
||||||
@ -241,6 +253,14 @@ func (m *mockDeployer) GetLogs(_ context.Context, _ string, _ int) (string, erro
|
|||||||
return m.logs, nil
|
return m.logs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockDeployer) AddIngressHost(_ context.Context, _, _ string) error {
|
||||||
|
return m.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockDeployer) RemoveIngressHost(_ context.Context, _, _ string) error {
|
||||||
|
return m.err
|
||||||
|
}
|
||||||
|
|
||||||
func setupInfraHandler() (*InfrastructureHandler, *mockGitRepository, *mockDNSProvider, *mockDeployer, chi.Router) {
|
func setupInfraHandler() (*InfrastructureHandler, *mockGitRepository, *mockDNSProvider, *mockDeployer, chi.Router) {
|
||||||
git := newMockGitRepository()
|
git := newMockGitRepository()
|
||||||
dns := newMockDNSProvider()
|
dns := newMockDNSProvider()
|
||||||
|
|||||||
@ -30,4 +30,13 @@ type Deployer interface {
|
|||||||
// GetLogs returns recent logs from the deployment pods.
|
// GetLogs returns recent logs from the deployment pods.
|
||||||
// tailLines specifies how many recent lines to return.
|
// tailLines specifies how many recent lines to return.
|
||||||
GetLogs(ctx context.Context, projectName string, tailLines int) (string, error)
|
GetLogs(ctx context.Context, projectName string, tailLines int) (string, error)
|
||||||
|
|
||||||
|
// AddIngressHost adds a new host to an existing project's ingress.
|
||||||
|
// This is used when adding domain aliases to a project.
|
||||||
|
// The host is added to both the TLS configuration and the routing rules.
|
||||||
|
AddIngressHost(ctx context.Context, projectName, host string) error
|
||||||
|
|
||||||
|
// RemoveIngressHost removes a host from an existing project's ingress.
|
||||||
|
// This is used when removing domain aliases from a project.
|
||||||
|
RemoveIngressHost(ctx context.Context, projectName, host string) error
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,6 +16,12 @@ type DNSProvider interface {
|
|||||||
// UpdateRecord updates an existing DNS record by ID.
|
// UpdateRecord updates an existing DNS record by ID.
|
||||||
UpdateRecord(ctx context.Context, recordID string, record domain.DNSRecord) (*domain.DNSRecord, error)
|
UpdateRecord(ctx context.Context, recordID string, record domain.DNSRecord) (*domain.DNSRecord, error)
|
||||||
|
|
||||||
|
// UpsertRecord creates or updates a DNS record.
|
||||||
|
// If a record with the same type and name exists, it updates it.
|
||||||
|
// Otherwise, it creates a new record.
|
||||||
|
// This is the preferred method for adding domain aliases where the record may already exist.
|
||||||
|
UpsertRecord(ctx context.Context, record domain.DNSRecord) (*domain.DNSRecord, error)
|
||||||
|
|
||||||
// DeleteRecord removes a DNS record by ID.
|
// DeleteRecord removes a DNS record by ID.
|
||||||
DeleteRecord(ctx context.Context, recordID string) error
|
DeleteRecord(ctx context.Context, recordID string) error
|
||||||
|
|
||||||
|
|||||||
@ -37,6 +37,7 @@ type ProjectInfraService struct {
|
|||||||
defaultGitOwner string
|
defaultGitOwner string
|
||||||
defaultDomain string
|
defaultDomain string
|
||||||
clusterIP string
|
clusterIP string
|
||||||
|
registryURL string // e.g., "registry.threesix.ai"
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProjectInfraConfig configures the project infrastructure service.
|
// ProjectInfraConfig configures the project infrastructure service.
|
||||||
@ -44,6 +45,7 @@ type ProjectInfraConfig struct {
|
|||||||
DefaultGitOwner string // e.g., "threesix"
|
DefaultGitOwner string // e.g., "threesix"
|
||||||
DefaultDomain string // e.g., "threesix.ai"
|
DefaultDomain string // e.g., "threesix.ai"
|
||||||
ClusterIP string // e.g., "208.122.204.172"
|
ClusterIP string // e.g., "208.122.204.172"
|
||||||
|
RegistryURL string // e.g., "registry.threesix.ai"
|
||||||
Logger *slog.Logger
|
Logger *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -63,6 +65,10 @@ func NewProjectInfraService(
|
|||||||
if logger == nil {
|
if logger == nil {
|
||||||
logger = slog.Default()
|
logger = slog.Default()
|
||||||
}
|
}
|
||||||
|
registryURL := cfg.RegistryURL
|
||||||
|
if registryURL == "" {
|
||||||
|
registryURL = "registry.threesix.ai" // Default for backward compatibility
|
||||||
|
}
|
||||||
return &ProjectInfraService{
|
return &ProjectInfraService{
|
||||||
db: db,
|
db: db,
|
||||||
gitRepo: gitRepo,
|
gitRepo: gitRepo,
|
||||||
@ -76,6 +82,7 @@ func NewProjectInfraService(
|
|||||||
defaultGitOwner: cfg.DefaultGitOwner,
|
defaultGitOwner: cfg.DefaultGitOwner,
|
||||||
defaultDomain: cfg.DefaultDomain,
|
defaultDomain: cfg.DefaultDomain,
|
||||||
clusterIP: cfg.ClusterIP,
|
clusterIP: cfg.ClusterIP,
|
||||||
|
registryURL: registryURL,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -66,7 +66,13 @@ func (s *ProjectInfraService) CreateProject(ctx context.Context, req CreateProje
|
|||||||
// 7. Seed repository with template
|
// 7. Seed repository with template
|
||||||
templateSeeded := s.seedTemplate(ctx, req, result)
|
templateSeeded := s.seedTemplate(ctx, req, result)
|
||||||
|
|
||||||
// 8. Trigger initial CI build if both CI and template are ready
|
// 8. Create initial K8s deployment (before triggering CI build)
|
||||||
|
// This ensures the deployment exists for `kubectl set image` in CI pipeline
|
||||||
|
if templateSeeded {
|
||||||
|
s.createInitialDeployment(ctx, req, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 9. Trigger initial CI build if both CI and template are ready
|
||||||
if ciActivated && templateSeeded && s.ciProvider != nil {
|
if ciActivated && templateSeeded && s.ciProvider != nil {
|
||||||
pipelineNum, err := s.ciProvider.TriggerBuild(ctx, result.GitRepoOwner, result.GitRepoName, "main")
|
pipelineNum, err := s.ciProvider.TriggerBuild(ctx, result.GitRepoOwner, result.GitRepoName, "main")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -287,6 +293,76 @@ func (s *ProjectInfraService) seedTemplate(ctx context.Context, req CreateProjec
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// createInitialDeployment creates the initial K8s deployment for a project.
|
||||||
|
// This is called after template seeding to ensure the deployment exists before
|
||||||
|
// the CI pipeline runs `kubectl set image`. The deployment will be in ImagePullBackOff
|
||||||
|
// until the first CI build completes and pushes the image.
|
||||||
|
func (s *ProjectInfraService) createInitialDeployment(ctx context.Context, req CreateProjectRequest, result *CreateProjectResult) {
|
||||||
|
if s.deployer == nil {
|
||||||
|
result.NextSteps = append(result.NextSteps, "Deployer not configured - run POST /projects/{id}/deploy after first build")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the expected image name that CI will push to
|
||||||
|
// Format: {registryURL}/{projectName}:latest
|
||||||
|
imageName := fmt.Sprintf("%s/%s:latest", s.registryURL, req.Name)
|
||||||
|
|
||||||
|
// Determine port based on template
|
||||||
|
port := templateDefaultPort(req.Template)
|
||||||
|
|
||||||
|
spec := domain.DeploySpec{
|
||||||
|
ProjectName: req.Name,
|
||||||
|
Image: imageName,
|
||||||
|
Domain: result.Domain,
|
||||||
|
Port: port,
|
||||||
|
Replicas: 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := s.deployer.Deploy(ctx, spec)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("failed to create initial deployment", "error", err, "project", req.Name)
|
||||||
|
result.NextSteps = append(result.NextSteps,
|
||||||
|
"Initial deployment failed - run POST /projects/{id}/deploy after first build completes",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("initial deployment created",
|
||||||
|
"project", req.Name,
|
||||||
|
"image", imageName,
|
||||||
|
"domain", result.Domain,
|
||||||
|
"note", "deployment will be pending until first CI build completes",
|
||||||
|
)
|
||||||
|
|
||||||
|
// Update database with deployment info
|
||||||
|
_, err = s.db.ExecContext(ctx, `
|
||||||
|
UPDATE projects SET
|
||||||
|
deployment_image = $1,
|
||||||
|
deployment_status = $2,
|
||||||
|
deployment_replicas = $3,
|
||||||
|
updated_at = $4
|
||||||
|
WHERE id = $5
|
||||||
|
`, imageName, "pending", 1, time.Now(), req.Name)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("failed to update project with deployment info", "error", err, "project", req.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// templateDefaultPort returns the default port for a template.
|
||||||
|
// Templates can override this by specifying a custom port in template metadata (future enhancement).
|
||||||
|
var templateDefaultPorts = map[string]int{
|
||||||
|
"astro-landing": 80, // nginx static server
|
||||||
|
"default": 80, // nginx static server
|
||||||
|
"go-api": 8080, // Go API server
|
||||||
|
}
|
||||||
|
|
||||||
|
func templateDefaultPort(templateName string) int {
|
||||||
|
if port, ok := templateDefaultPorts[templateName]; ok {
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
return 80 // Default to nginx port for static sites
|
||||||
|
}
|
||||||
|
|
||||||
// GetStatus returns the current status of a project.
|
// GetStatus returns the current status of a project.
|
||||||
func (s *ProjectInfraService) GetStatus(ctx context.Context, projectID string) (*ProjectStatus, error) {
|
func (s *ProjectInfraService) GetStatus(ctx context.Context, projectID string) (*ProjectStatus, error) {
|
||||||
var status ProjectStatus
|
var status ProjectStatus
|
||||||
|
|||||||
@ -61,7 +61,7 @@ func (s *ProjectInfraService) AddDomain(ctx context.Context, req AddDomainReques
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dnsRecord, err := s.dns.CreateRecord(ctx, domain.DNSRecord{
|
dnsRecord, err := s.dns.UpsertRecord(ctx, domain.DNSRecord{
|
||||||
Type: recordType,
|
Type: recordType,
|
||||||
Name: subdomain,
|
Name: subdomain,
|
||||||
Content: content,
|
Content: content,
|
||||||
@ -69,7 +69,7 @@ func (s *ProjectInfraService) AddDomain(ctx context.Context, req AddDomainReques
|
|||||||
Proxied: req.Proxied,
|
Proxied: req.Proxied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create DNS record: %w", err)
|
return nil, fmt.Errorf("failed to upsert DNS record: %w", err)
|
||||||
}
|
}
|
||||||
pd.DNSRecordID = dnsRecord.ID
|
pd.DNSRecordID = dnsRecord.ID
|
||||||
pd.Verified = true
|
pd.Verified = true
|
||||||
@ -84,6 +84,14 @@ func (s *ProjectInfraService) AddDomain(ctx context.Context, req AddDomainReques
|
|||||||
return nil, fmt.Errorf("failed to store domain: %w", err)
|
return nil, fmt.Errorf("failed to store domain: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add host to K8s ingress (for both threesix.ai and external domains)
|
||||||
|
if s.deployer != nil {
|
||||||
|
if err := s.deployer.AddIngressHost(ctx, req.ProjectID, req.Domain); err != nil {
|
||||||
|
s.logger.Warn("failed to add ingress host", "error", err, "project", req.ProjectID, "domain", req.Domain)
|
||||||
|
// Don't fail the request - DNS and DB are already set up
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
s.logger.Info("domain added", "project", req.ProjectID, "domain", req.Domain, "type", domainType)
|
s.logger.Info("domain added", "project", req.ProjectID, "domain", req.Domain, "type", domainType)
|
||||||
return pd, nil
|
return pd, nil
|
||||||
}
|
}
|
||||||
@ -135,6 +143,14 @@ func (s *ProjectInfraService) RemoveDomain(ctx context.Context, projectID, fqdn
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Remove host from K8s ingress
|
||||||
|
if s.deployer != nil {
|
||||||
|
if err := s.deployer.RemoveIngressHost(ctx, projectID, fqdn); err != nil {
|
||||||
|
s.logger.Warn("failed to remove ingress host", "error", err, "project", projectID, "domain", fqdn)
|
||||||
|
// Continue anyway - DNS record is already deleted
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Delete from database
|
// Delete from database
|
||||||
if err := s.domainRepo.Delete(ctx, pd.ID); err != nil {
|
if err := s.domainRepo.Delete(ctx, pd.ID); err != nil {
|
||||||
return fmt.Errorf("failed to delete domain: %w", err)
|
return fmt.Errorf("failed to delete domain: %w", err)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user