# Install auditd (most Linux distributions include it)
sudo apt-get install -y auditd audispd-plugins  # Ubuntu/Debian
sudo yum install -y audit audit-libs            # CentOS/Rocky

sudo systemctl enable --now auditd

# Configure audit rules - comprehensive security monitoring
cat > /etc/audit/rules.d/audit.rules <<EOF
# Delete all existing rules
-D

# Increase buffer size for high-load systems
-b 8192

# Failure mode: 1=print failure, 2=panic
-f 1

# Monitor authentication
-w /etc/pam.d/ -p wa -k pam_changes
-w /etc/passwd -p wa -k user_modification
-w /etc/shadow -p wa -k user_modification
-w /etc/group -p wa -k group_modification
-w /etc/sudoers -p wa -k sudoers_changes
-w /etc/sudoers.d/ -p wa -k sudoers_changes

# Login/logout events
-w /var/log/lastlog -p wa -k login_events
-w /var/run/faillock/ -p wa -k login_failures

# Privilege escalation
-a always,exit -F arch=b64 -S setuid -S setgid -F exit=0 -k setuid
-a always,exit -F arch=b64 -S execve -C uid!=euid -F euid=0 -k privilege_escalation

# File system mounts
-a always,exit -F arch=b64 -S mount -k filesystem_mounts
-a always,exit -F arch=b64 -S umount2 -k filesystem_umounts

# Network configuration changes
-w /etc/hosts -p wa -k network_modification
-w /etc/network/ -p wa -k network_modification
-w /etc/sysconfig/network -p wa -k network_modification
-a always,exit -F arch=b64 -S sethostname -S setdomainname -k network_modification

# Kernel module loading
-w /sbin/insmod -p x -k kernel_module
-w /sbin/rmmod -p x -k kernel_module
-w /sbin/modprobe -p x -k kernel_module
-a always,exit -F arch=b64 -S init_module -k kernel_module

# System time changes
-a always,exit -F arch=b64 -S adjtimex -S settimeofday -k time_change
-a always,exit -F arch=b64 -S clock_settime -k time_change
-w /etc/localtime -p wa -k time_change

# File deletion
-a always,exit -F arch=b64 -S unlink -S unlinkat -S rename -S renameat -k file_deletion

# SSH access
-w /etc/ssh/sshd_config -p wa -k sshd_config
-w /root/.ssh/ -p wa -k root_ssh_keys

# Make rules immutable (requires reboot to change - important for compliance)
-e 2
EOF

sudo augenrules --load
sudo systemctl restart auditd

# Verify rules are loaded
sudo auditctl -l
sudo auditctl -s  # Status

# Create a dedicated partition for audit logs
# In /etc/fstab, mount with noexec,nosuid options:
# /dev/sdb1 /var/log/audit ext4 defaults,noexec,nosuid,nodev 0 0

# Set append-only attribute on audit log files (chattr)
sudo chattr +a /var/log/audit/audit.log

# Verify immutable attribute
sudo lsattr /var/log/audit/audit.log
# Should show: -----a--------e---- /var/log/audit/audit.log

# For logs in /var/log, use chattr on the directory
sudo chattr +a /var/log/audit/

# Note: chattr +a allows appending but not deletion or modification
# Even root cannot delete files with the +a attribute set
# Only a user with CAP_LINUX_IMMUTABLE can change this

# Set up dedicated audit log partition with journal mode
# This provides an additional barrier against log tampering
sudo tune2fs -E lazy_itable_init=0 /dev/sdb1
sudo mount -o remount,journal_async_commit /var/log/audit

# Edit /etc/audit/auditd.conf for retention settings
cat > /etc/audit/auditd.conf <<EOF
log_file = /var/log/audit/audit.log
log_format = RAW
log_group = root
priority_boost = 4
flush = INCREMENTAL_ASYNC
freq = 50
num_logs = 12              # Keep 12 rotated files
max_log_file = 256         # Max log file size in MB
max_log_file_action = ROTATE
space_left = 500           # Warn when 500MB remaining
space_left_action = SYSLOG
admin_space_left = 50      # Critical threshold in MB
admin_space_left_action = HALT  # Stop system if log full (prevents audit loss)
disk_full_action = HALT
disk_error_action = SYSLOG
use_libwrap = yes
tcp_listen_queue = 5
tcp_max_per_addr = 1
tcp_client_max_idle = 0
transport = TCP
krb5_principal = auditd
distribute_network = no
EOF

sudo systemctl restart auditd

# Set up log rotation with logrotate
cat > /etc/logrotate.d/audit <<EOF
/var/log/audit/audit.log {
    daily
    rotate 365           # Keep 365 rotated files (1 year local)
    compress
    delaycompress
    missingok
    notifempty
    create 0600 root root
    postrotate
        /sbin/service auditd restart 2> /dev/null > /dev/null || true
    endscript
}
EOF

# Create audit log archival script
cat > /usr/local/bin/audit-archive.sh <<'EOF'
#!/bin/bash
# Archive audit logs to S3 with integrity hash

set -euo pipefail

S3_BUCKET="s3://company-audit-logs"
LOCAL_ARCHIVE="/var/log/audit/archive"
HASH_LOG="/var/log/audit/archive/checksums.sha256"
DATE=$(date +%Y/%m/%d)
TIMESTAMP=$(date +%Y%m%d_%H%M%S)

mkdir -p "${LOCAL_ARCHIVE}"

# Collect logs older than 1 day
find /var/log/audit -name "audit.log.*" -mtime +1 | while read -r logfile; do
  BASENAME=$(basename "$logfile")
  ARCHIVE_NAME="${TIMESTAMP}_${BASENAME}.gz"
  
  # Compress and hash
  gzip -c "$logfile" > "${LOCAL_ARCHIVE}/${ARCHIVE_NAME}"
  
  # Generate SHA-256 checksum
  sha256sum "${LOCAL_ARCHIVE}/${ARCHIVE_NAME}" >> "${HASH_LOG}"
  
  # Upload to S3 with server-side encryption
  aws s3 cp \
    "${LOCAL_ARCHIVE}/${ARCHIVE_NAME}" \
    "${S3_BUCKET}/${DATE}/${ARCHIVE_NAME}" \
    --sse aws:kms \
    --sse-kms-key-id "arn:aws:kms:us-east-1:123456789:key/your-key-id" \
    --metadata "sha256=$(sha256sum ${LOCAL_ARCHIVE}/${ARCHIVE_NAME} | cut -d' ' -f1),hostname=$(hostname),archived=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  
  echo "Archived: $logfile -> ${S3_BUCKET}/${DATE}/${ARCHIVE_NAME}"
done

# Upload checksum file
aws s3 cp "${HASH_LOG}" \
  "${S3_BUCKET}/${DATE}/checksums.sha256" \
  --sse aws:kms

echo "Archive complete."
EOF

chmod +x /usr/local/bin/audit-archive.sh

# Schedule nightly archival
echo "0 1 * * * root /usr/local/bin/audit-archive.sh >> /var/log/audit-archive.log 2>&1" \
  > /etc/cron.d/audit-archive

# Create S3 lifecycle policy
cat > audit-lifecycle.json <<EOF
{
  "Rules": [
    {
      "ID": "AuditLogRetention",
      "Status": "Enabled",
      "Prefix": "",
      "Transitions": [
        {
          "Days": 90,
          "StorageClass": "STANDARD_IA"
        },
        {
          "Days": 365,
          "StorageClass": "GLACIER"
        },
        {
          "Days": 1095,
          "StorageClass": "DEEP_ARCHIVE"
        }
      ],
      "Expiration": {
        "Days": 2557  # 7 years (SOX/HIPAA requirement)
      },
      "NoncurrentVersionExpiration": {
        "NoncurrentDays": 30
      }
    }
  ]
}
EOF

# Apply lifecycle policy
aws s3api put-bucket-lifecycle-configuration \
  --bucket company-audit-logs \
  --lifecycle-configuration file://audit-lifecycle.json

# Enable S3 Object Lock for WORM (Write Once Read Many) compliance
aws s3api put-object-lock-configuration \
  --bucket company-audit-logs \
  --object-lock-configuration '{
    "ObjectLockEnabled": "Enabled",
    "Rule": {
      "DefaultRetention": {
        "Mode": "COMPLIANCE",
        "Years": 7
      }
    }
  }'

# Enable versioning (required for Object Lock)
aws s3api put-bucket-versioning \
  --bucket company-audit-logs \
  --versioning-configuration Status=Enabled

# PCI-DSS Requirements:
# - Retain audit logs for at least 1 year
# - 3 months immediately available, 9 months archived
# - Daily log review for all in-scope systems

# HIPAA Requirements:
# - 6 years retention
# - Must be able to determine who accessed PHI and when
# Required auditd rules for HIPAA:
cat >> /etc/audit/rules.d/hipaa.rules <<EOF
# Monitor access to files containing PHI
-w /var/data/patient-records/ -p warx -k phi_access
-w /var/backups/patient-data/ -p r -k phi_access

# Monitor database audit trails
-w /var/lib/mysql/ -p wa -k db_changes
-w /var/lib/postgresql/ -p wa -k db_changes

# Track administrative access
-a always,exit -F arch=b64 -S all -F euid=0 -k admin_actions
EOF

# SOX Requirements:
# - Financial data access and modification must be logged
# - Logs must be retained per company policy (typically 7 years)
# - Quarterly review and attestation

# Generate a compliance report
cat > /usr/local/bin/audit-compliance-report.sh <<'EOF'
#!/bin/bash
REPORT_DATE=$(date +%Y-%m-%d)
REPORT_FILE="/var/log/compliance/audit-report-${REPORT_DATE}.txt"
mkdir -p /var/log/compliance

echo "=== Audit Log Compliance Report: $REPORT_DATE ===" > "$REPORT_FILE"
echo "" >> "$REPORT_FILE"

echo "=== Authentication Events (Last 24h) ===" >> "$REPORT_FILE"
ausearch -k pam_changes --start today 2>/dev/null | aureport -i >> "$REPORT_FILE"

echo "" >> "$REPORT_FILE"
echo "=== Privilege Escalation (Last 24h) ===" >> "$REPORT_FILE"
ausearch -k privilege_escalation --start today 2>/dev/null >> "$REPORT_FILE"

echo "" >> "$REPORT_FILE"
echo "=== Failed Login Attempts (Last 24h) ===" >> "$REPORT_FILE"
ausearch -k login_failures --start today 2>/dev/null >> "$REPORT_FILE"

echo "" >> "$REPORT_FILE"
echo "=== File System Changes (Last 24h) ===" >> "$REPORT_FILE"
ausearch -k sudoers_changes --start today 2>/dev/null >> "$REPORT_FILE"

echo "Report generated: $REPORT_FILE"
EOF
chmod +x /usr/local/bin/audit-compliance-report.sh

# Verify archived logs have not been tampered with
cat > /usr/local/bin/verify-audit-integrity.sh <<'EOF'
#!/bin/bash
S3_BUCKET="s3://company-audit-logs"
VERIFY_DATE="${1:-$(date +%Y/%m/%d)}"
TEMP_DIR=$(mktemp -d)

echo "Verifying audit log integrity for: $VERIFY_DATE"

# Download checksums file
aws s3 cp "${S3_BUCKET}/${VERIFY_DATE}/checksums.sha256" "${TEMP_DIR}/checksums.sha256"

# Download and verify each log file
while IFS=" " read -r checksum filename; do
  BASENAME=$(basename "$filename")
  aws s3 cp "${S3_BUCKET}/${VERIFY_DATE}/${BASENAME}" "${TEMP_DIR}/${BASENAME}"
  
  ACTUAL=$(sha256sum "${TEMP_DIR}/${BASENAME}" | cut -d' ' -f1)
  
  if [ "$checksum" = "$ACTUAL" ]; then
    echo "OK: $BASENAME"
  else
    echo "TAMPERED: $BASENAME"
    echo "  Expected: $checksum"
    echo "  Actual:   $ACTUAL"
    exit 1
  fi
done < "${TEMP_DIR}/checksums.sha256"

rm -rf "$TEMP_DIR"
echo "All audit logs verified successfully."
EOF

chmod +x /usr/local/bin/verify-audit-integrity.sh

# Schedule weekly integrity check
echo "0 3 * * 0 root /usr/local/bin/verify-audit-integrity.sh >> /var/log/integrity-check.log 2>&1" \
  > /etc/cron.d/audit-integrity

# Search audit logs with ausearch
# Find all sudo commands by a specific user
ausearch -ua john.doe -k privilege_escalation --start 2024-01-01 --end 2024-01-31

# Find all failed logins
ausearch -m USER_AUTH -sv no --start today

# Find file access to sensitive files
ausearch -f /etc/shadow --start yesterday

# Generate login summary report
aureport --login --start 2024-01-01 --end 2024-01-31

# Generate failed events report
aureport --failed --start this-month

# Search archived logs in S3
# Download date range for investigation
for day in 01 02 03 04 05; do
  aws s3 sync \
    "s3://company-audit-logs/2024/01/${day}/" \
    "/tmp/audit-investigation/2024-01-${day}/"
done

# Search decompressed archives
find /tmp/audit-investigation -name "*.gz" -exec zcat {} \; | \
  ausearch -i --input - -f /etc/passwd

# Parse with Python for custom analysis
python3 << 'PYEOF'
import json
import subprocess

# Get audit events as JSON
result = subprocess.run(
    ['ausearch', '-m', 'USER_CMD', '--start', 'this-month', '--format', 'enriched'],
    capture_output=True, text=True
)

# Parse and count sudo commands by user
commands = {}
for line in result.stdout.split('\n'):
    if 'acct=' in line and 'cmd=' in line:
        parts = dict(kv.split('=', 1) for kv in line.split() if '=' in kv)
        user = parts.get('acct', 'unknown').strip('"')
        commands[user] = commands.get(user, 0) + 1

for user, count in sorted(commands.items(), key=lambda x: -x[1]):
    print(f"{user}: {count} sudo commands")
PYEOF

# Check auditd status
sudo systemctl status auditd
sudo auditctl -s

# Check disk space
df -h /var/log/audit/

# Verify audit rules are loaded
sudo auditctl -l | head -20

# Test S3 connectivity
aws s3 ls s3://company-audit-logs/

# Check IAM permissions
aws s3api put-object --bucket company-audit-logs --key test/test.txt --body /dev/null
aws s3api delete-object --bucket company-audit-logs --key test/test.txt

# Check archive script logs
sudo tail -50 /var/log/audit-archive.log

# Reduce log verbosity temporarily
sudo auditctl -e 0  # Disable auditing (emergency only)

# Archive old logs immediately
sudo /usr/local/bin/audit-archive.sh

# Increase disk space or adjust max_log_file in auditd.conf
sudo vim /etc/audit/auditd.conf
sudo systemctl restart auditd