Production DevSecOps Pipeline — The Complete Day-2 Operations Runbook

# AWS CLI v2
winget install Amazon.AWSCLI

# Terraform 1.6+
winget install HashiCorp.Terraform

# Terragrunt
# Download from https://github.com/gruntwork-io/terragrunt/releases
# Place in C:\Windows\System32\ or add to PATH

# kubectl
winget install Kubernetes.kubectl

# ArgoCD CLI
winget install argoproj.argocd

[profile myapp-production-use1]
region = us-east-1
role_arn = arn:aws:iam::591120834781:role/AdministratorAccess
source_profile = default

[profile myapp-production-usw2]
region = us-west-2
role_arn = arn:aws:iam::591120834781:role/AdministratorAccess
source_profile = default

[profile myapp-staging-use1]
region = us-east-1
role_arn = arn:aws:iam::690687753178:role/AdministratorAccess
source_profile = default

[profile myapp-staging-usw2]
region = us-west-2
role_arn = arn:aws:iam::690687753178:role/AdministratorAccess
source_profile = default

[profile myapp-dev-use1]
region = us-east-1
role_arn = arn:aws:iam::557702566877:role/AdministratorAccess
source_profile = default

[profile myapp-dev-usw2]
region = us-west-2
role_arn = arn:aws:iam::557702566877:role/AdministratorAccess
source_profile = default

# Production us-east-1
terragrunt apply --terragrunt-working-dir live/production/us-east-1/waf
# Output → webacl_arn  (copy this value)

# Production us-west-2
terragrunt apply --terragrunt-working-dir live/production/us-west-2/waf
# Output → webacl_arn  (copy this value)

# Staging (no GitOps ARN needed, but good to have)
terragrunt apply --terragrunt-working-dir live/staging/us-east-1/waf
terragrunt apply --terragrunt-working-dir live/staging/us-west-2/waf

terragrunt apply --terragrunt-working-dir live/production/us-east-1/guardduty
terragrunt apply --terragrunt-working-dir live/production/us-west-2/guardduty
terragrunt apply --terragrunt-working-dir live/staging/us-east-1/guardduty
terragrunt apply --terragrunt-working-dir live/staging/us-west-2/guardduty

# Staging us-east-1
terragrunt apply --terragrunt-working-dir live/staging/us-east-1/eso-irsa
# Output → role_arn  (copy → used in environments/staging/applicationset.yaml)

# Staging us-west-2
terragrunt apply --terragrunt-working-dir live/staging/us-west-2/eso-irsa
# Output → role_arn  (copy → used in environments/staging/applicationset.yaml)

terragrunt apply --terragrunt-working-dir live/production/us-east-1/fluent-bit-irsa
# → role_arn for myapp-production-use1

terragrunt apply --terragrunt-working-dir live/production/us-west-2/fluent-bit-irsa
# → role_arn for myapp-production-usw2

terragrunt apply --terragrunt-working-dir live/staging/us-east-1/fluent-bit-irsa
# → role_arn for myapp-staging-use1

terragrunt apply --terragrunt-working-dir live/staging/us-west-2/fluent-bit-irsa
# → role_arn for myapp-staging-usw2

terragrunt apply --terragrunt-working-dir live/dev/us-east-1/fluent-bit-irsa
# → role_arn for myapp-dev-use1

terragrunt apply --terragrunt-working-dir live/dev/us-west-2/fluent-bit-irsa
# → role_arn for myapp-dev-usw2

terragrunt apply --terragrunt-working-dir live/production/us-east-1/karpenter
# Outputs:
#   controller_role_arn   → for karpenter applicationset.yaml
#   node_role_arn         → for verification (name = myapp-production-use1-karpenter-node)
#   node_instance_profile → for verification
#   interruption_queue_name → should be "myapp-production-use1-karpenter"

terragrunt apply --terragrunt-working-dir live/production/us-west-2/karpenter
# Outputs same structure for usw2

# Production
terragrunt apply --terragrunt-working-dir live/production/us-east-1/velero
# → role_arn for myapp-production-use1

terragrunt apply --terragrunt-working-dir live/production/us-west-2/velero
# → role_arn for myapp-production-usw2

# Staging
terragrunt apply --terragrunt-working-dir live/staging/us-east-1/velero
# → role_arn for myapp-staging-use1

terragrunt apply --terragrunt-working-dir live/staging/us-west-2/velero
# → role_arn for myapp-staging-usw2

# Dev
terragrunt apply --terragrunt-working-dir live/dev/us-east-1/velero
# → role_arn for myapp-dev-use1

terragrunt apply --terragrunt-working-dir live/dev/us-west-2/velero
# → role_arn for myapp-dev-usw2

elements:
  - cluster: myapp-production-use1
    ...
    wafAclArn: "arn:aws:wafv2:us-east-1:591120834781:regional/webacl/myapp-production-use1-waf/XXXXXXXX"
  - cluster: myapp-production-usw2
    ...
    wafAclArn: "arn:aws:wafv2:us-west-2:591120834781:regional/webacl/myapp-production-usw2-waf/XXXXXXXX"

elements:
  - cluster: myapp-staging-use1
    ...
    irsaRoleArn: "arn:aws:iam::690687753178:role/myapp-staging-use1-eso"
  - cluster: myapp-staging-usw2
    ...
    irsaRoleArn: "arn:aws:iam::690687753178:role/myapp-staging-usw2-eso"

elements:
  - cluster: myapp-production-use1  roleArn: "arn:aws:iam::591120834781:role/myapp-production-use1-fluent-bit"
  - cluster: myapp-production-usw2  roleArn: "arn:aws:iam::591120834781:role/myapp-production-usw2-fluent-bit"
  - cluster: myapp-staging-use1     roleArn: "arn:aws:iam::690687753178:role/myapp-staging-use1-fluent-bit"
  - cluster: myapp-staging-usw2     roleArn: "arn:aws:iam::690687753178:role/myapp-staging-usw2-fluent-bit"
  - cluster: myapp-dev-use1         roleArn: "arn:aws:iam::557702566877:role/myapp-dev-use1-fluent-bit"
  - cluster: myapp-dev-usw2         roleArn: "arn:aws:iam::557702566877:role/myapp-dev-usw2-fluent-bit"

elements:
  - cluster: myapp-production-use1  controllerRole: "arn:aws:iam::591120834781:role/myapp-production-use1-karpenter"
  - cluster: myapp-production-usw2  controllerRole: "arn:aws:iam::591120834781:role/myapp-production-usw2-karpenter"

elements:
  - cluster: myapp-production-use1  roleArn: "arn:aws:iam::591120834781:role/myapp-production-use1-velero"
  - cluster: myapp-production-usw2  roleArn: "arn:aws:iam::591120834781:role/myapp-production-usw2-velero"
  - cluster: myapp-staging-use1     roleArn: "arn:aws:iam::690687753178:role/myapp-staging-use1-velero"
  - cluster: myapp-staging-usw2     roleArn: "arn:aws:iam::690687753178:role/myapp-staging-usw2-velero"
  - cluster: myapp-dev-use1         roleArn: "arn:aws:iam::557702566877:role/myapp-dev-use1-velero"
  - cluster: myapp-dev-usw2         roleArn: "arn:aws:iam::557702566877:role/myapp-dev-usw2-velero"

cd myapp-gitops
git add environments/ infrastructure/
git commit -m "chore: fill in real ARNs from terraform outputs"
git push origin HEAD:main

AWS_PROFILE=myapp-staging-use1 aws secretsmanager create-secret \
  --name staging/myapp/db-password \
  --secret-string '{"password":"change-me-staging"}' \
  --region us-east-1

AWS_PROFILE=myapp-staging-usw2 aws secretsmanager create-secret \
  --name staging/myapp/db-password \
  --secret-string '{"password":"change-me-staging"}' \
  --region us-west-2

# Point kubectl at production cluster (where ArgoCD runs)
kubectl config use-context myapp-production-use1

cd myapp-gitops

# One-time bootstrap — creates the self-managing Application
kubectl apply -f argocd/bootstrap.yaml -n argocd

# ArgoCD will now sync argocd/project-production.yaml automatically.
# Watch until it's healthy:
argocd app wait bootstrap --health

cd myapp-gitops

kubectl apply -f infrastructure/eso/applicationset.yaml
kubectl apply -f infrastructure/monitoring/applicationset.yaml
kubectl apply -f infrastructure/monitoring/alert-rules-applicationset.yaml
kubectl apply -f infrastructure/logging/applicationset.yaml
kubectl apply -f infrastructure/karpenter/applicationset.yaml
kubectl apply -f infrastructure/karpenter/nodepool-applicationset.yaml
kubectl apply -f infrastructure/velero/applicationset.yaml
kubectl apply -f infrastructure/falco/applicationset.yaml
kubectl apply -f infrastructure/argo-rollouts/applicationset.yaml

# Step 1: Prometheus stack (creates CRDs for PrometheusRule, ServiceMonitor, etc.)
argocd app sync prometheus-myapp-production-use1 prometheus-myapp-production-usw2
argocd app wait prometheus-myapp-production-use1 --health
argocd app wait prometheus-myapp-production-usw2 --health

# Step 2: Alert rules (needs Prometheus CRDs)
argocd app sync alert-rules-myapp-production-use1 alert-rules-myapp-production-usw2

# Step 3: Parallel infra components (no inter-dependency)
argocd app sync \
  fluent-bit-myapp-production-use1 fluent-bit-myapp-production-usw2 \
  velero-myapp-production-use1 velero-myapp-production-usw2 \
  falco-myapp-production-use1 falco-myapp-production-usw2

# Step 4: Karpenter controller (needs ECR access to pull image from public.ecr.aws)
argocd app sync karpenter-myapp-production-use1 karpenter-myapp-production-usw2
argocd app wait karpenter-myapp-production-use1 --health

# Step 5: Karpenter NodePools (needs Karpenter CRDs installed by Step 4)
argocd app sync karpenter-nodepool-myapp-production-use1 karpenter-nodepool-myapp-production-usw2

# Step 6: Argo Rollouts controller
argocd app sync argo-rollouts-myapp-production-use1 argo-rollouts-myapp-production-usw2
argocd app wait argo-rollouts-myapp-production-use1 --health

# Step 7: App (uses Rollout CR — needs argo-rollouts controller running)
argocd app sync myapp-production-myapp-production-use1 myapp-production-myapp-production-usw2

argocd app sync \
  eso-myapp-staging-use1 eso-myapp-staging-usw2 \
  fluent-bit-myapp-staging-use1 fluent-bit-myapp-staging-usw2 \
  velero-myapp-staging-use1 velero-myapp-staging-usw2 \
  falco-myapp-staging-use1 falco-myapp-staging-usw2 \
  prometheus-myapp-staging-use1 prometheus-myapp-staging-usw2

# After staging ESO is healthy, ExternalSecrets will sync automatically
argocd app sync myapp-staging-myapp-staging-use1 myapp-staging-myapp-staging-usw2

kubectl get pods -n monitoring --context myapp-production-use1
kubectl get prometheusrule -n monitoring --context myapp-production-use1
kubectl get alertmanager -n monitoring --context myapp-production-use1
# Access Grafana: kubectl port-forward svc/kube-prometheus-stack-grafana 3000:80 -n monitoring

kubectl get pods -n logging --context myapp-production-use1
# Verify log groups were created:
AWS_PROFILE=myapp-production-use1 aws logs describe-log-groups \
  --log-group-name-prefix /eks/myapp-production-use1 --region us-east-1

kubectl get pods -n karpenter --context myapp-production-use1
kubectl get nodepool --context myapp-production-use1
kubectl get ec2nodeclass --context myapp-production-use1
# Trigger a scale test:
kubectl scale deploy/stress --replicas=50 -n default --context myapp-production-use1
kubectl get nodes -w --context myapp-production-use1

kubectl get pods -n velero --context myapp-production-use1
kubectl get schedule -n velero --context myapp-production-use1
# Trigger manual backup:
velero backup create manual-test --context myapp-production-use1
velero backup describe manual-test --context myapp-production-use1

kubectl get pods -n falco --context myapp-production-use1
# Check CloudWatch for events:
AWS_PROFILE=myapp-production-use1 aws logs describe-log-groups \
  --log-group-name-prefix /falco --region us-east-1

kubectl get rollout -n production --context myapp-production-use1
kubectl argo rollouts get rollout myapp-production-use1-myapp -n production \
  --context myapp-production-use1 --watch

kubectl get externalsecret -n staging --context myapp-staging-use1
kubectl describe externalsecret myapp-production-use1-myapp-secrets -n staging \
  --context myapp-staging-use1

AWS_PROFILE=myapp-production-use1 aws wafv2 list-web-acls \
  --scope REGIONAL --region us-east-1 | grep myapp

AWS_PROFILE=myapp-production-use1 aws guardduty list-detectors --region us-east-1
AWS_PROFILE=myapp-production-usw2 aws guardduty list-detectors --region us-west-2

# 1. Authenticate
aws sso login --sso-session admin --no-browser
# → browser opens → click Allow → wait for "Successfully logged in"

# 2. Verify
aws sts get-caller-identity --profile myapp-prod-use1
# → should return Account: 591120834781

# 3. Quick health check
curl https://api.matthewoladipupo.dev/health
# → {"status":"healthy","region":"us-east-1"}

# Enable (wait ~3 minutes after running this)
aws eks update-cluster-config \
  --name myapp-production-use1 \
  --region us-east-1 \
  --profile myapp-prod-use1 \
  --resources-vpc-config endpointPublicAccess=true,endpointPrivateAccess=true,publicAccessCidrs=0.0.0.0/0

# Confirm it's ready
aws eks describe-cluster \
  --name myapp-production-use1 \
  --region us-east-1 \
  --profile myapp-prod-use1 \
  --query 'cluster.resourcesVpcConfig.endpointPublicAccess'
# Must return: true

# --- Do your kubectl work here ---

# Lock back immediately after
aws eks update-cluster-config \
  --name myapp-production-use1 \
  --region us-east-1 \
  --profile myapp-prod-use1 \
  --resources-vpc-config endpointPublicAccess=false,endpointPrivateAccess=true

# Watch rollout steps (enable public endpoint first)
kubectl get rollouts -n production --context myapp-production-use1 -w

# Detailed view (requires kubectl-argo-rollouts plugin)
kubectl argo rollouts get rollout myapp -n production \
  --context myapp-production-use1 --watch

kubectl argo rollouts promote myapp -n production --context myapp-production-use1

kubectl argo rollouts abort myapp -n production --context myapp-production-use1

# After abort the rollout shows Degraded — retry to return to Healthy
kubectl argo rollouts retry rollout myapp -n production --context myapp-production-use1

cd /path/to/myapp-gitops
git revert HEAD --no-edit
git push origin main
# ArgoCD auto-syncs the revert within 3 minutes

argocd app history myapp-production-myapp-production-use1
argocd app rollback myapp-production-myapp-production-use1 <revision-number>

# Get previous image tag from ECR
aws ecr describe-images \
  --repository-name myapp \
  --region us-east-1 \
  --profile myapp-prod-use1 \
  --query 'sort_by(imageDetails,&imagePushedAt)[-2].imageTags' \
  --output json

# Force update the rollout
kubectl argo rollouts set image myapp \
  myapp=206617159586.dkr.ecr.us-east-1.amazonaws.com/myapp:sha-<previous-sha> \
  -n production --context myapp-production-use1

# After incident: update values-production.yaml in gitops to match, then push

aws secretsmanager put-secret-value \
  --secret-id production/myapp/db-password \
  --secret-string '{"password":"new-value-here"}' \
  --region us-east-1 \
  --profile myapp-prod-use1

# Enable public endpoint first (OPS-2)
kubectl annotate externalsecret myapp-db-password \
  force-sync=$(date +%s) -n production \
  --context myapp-production-use1 --overwrite

# Verify
kubectl get externalsecret myapp-db-password -n production \
  --context myapp-production-use1
# STATUS: SecretSynced

kubectl argo rollouts restart myapp -n production --context myapp-production-use1

# 1. Identify what triggered the alert
kubectl logs -n falco daemonset/falco --context myapp-production-use1 \
  | grep -E "Warning|Critical" | tail -20

# 2. Inspect the affected pod
kubectl describe pod <pod-name> -n production --context myapp-production-use1
kubectl logs <pod-name> -n production --context myapp-production-use1 --tail=100

# 3. Contain if confirmed malicious — delete pod (Rollout replaces with clean copy)
kubectl delete pod <pod-name> -n production --context myapp-production-use1

# 4. Preserve logs before deletion
kubectl logs <pod-name> -n production --context myapp-production-use1 \
  > /tmp/incident-$(date +%Y%m%d-%H%M%S).log

# 5. Check GuardDuty for correlated findings
aws guardduty list-findings \
  --detector-id $(aws guardduty list-detectors \
    --region us-east-1 --profile myapp-prod-use1 \
    --query 'DetectorIds[0]' --output text) \
  --region us-east-1 --profile myapp-prod-use1

# See the rejection reason
kubectl describe pod <pod-name> -n <namespace> --context myapp-production-use1
# Look for: "admission webhook" in Events

# List policy violations
kubectl get policyreport -n <namespace> --context myapp-production-use1

# Check pods
kubectl get pods -n production --context myapp-production-use1

# Check pod logs
kubectl logs -n production -l app=myapp --context myapp-production-use1 --tail=50

# Check rollout
kubectl get rollouts -n production --context myapp-production-use1

# Check HPA — is it maxed out?
kubectl get hpa -n production --context myapp-production-use1

# Check if Karpenter is provisioning nodes
kubectl get nodes --context myapp-production-use1 -w

# Application
curl https://api.matthewoladipupo.dev/health

# ArgoCD — show only non-Synced apps (empty = all good)
argocd app list | grep -v "Synced.*Healthy"

# Enable public endpoint, then:
kubectl get nodes --context myapp-production-use1
kubectl get pods -n production --context myapp-production-use1
kubectl get hpa -n production --context myapp-production-use1
kubectl get externalsecret -n production --context myapp-production-use1
kubectl get schedule -n velero --context myapp-production-use1

# Lock endpoint back

# Enable public endpoint first (OPS-2), then:

# List available backups
kubectl get backups -n velero --context myapp-production-use1

# Restore a namespace
kubectl create -f - <<EOF
apiVersion: velero.io/v1
kind: Restore
metadata:
  name: restore-$(date +%Y%m%d-%H%M)
  namespace: velero
spec:
  backupName: <backup-name-from-list>
  includedNamespaces:
    - production
  restorePVs: true
EOF

# Watch progress
kubectl get restore -n velero --context myapp-production-use1 -w