/etc/systemd/system/        # Admin-created, highest priority
/run/systemd/system/        # Runtime units (not persistent)
/lib/systemd/system/        # Package-installed units (lowest priority)
/usr/lib/systemd/system/    # Same as above on some distros

# Show all unit files and their state
systemctl list-unit-files

# Show all running units
systemctl list-units --state=running

# Show unit file content and drop-in files
systemctl cat nginx.service

# Show all properties of a unit
systemctl show nginx.service

# Create a custom service
sudo nano /etc/systemd/system/myapp.service

[Unit]
Description=My Application Server
Documentation=https://docs.example.com
# This unit requires network to be up
Requires=network.target
# Start after network but don't hard-require it
After=network.target postgresql.service
# Conditional start
ConditionPathExists=/opt/myapp/app.py

[Service]
Type=simple
# Run as non-root user
User=myapp
Group=myapp
WorkingDirectory=/opt/myapp

# Environment variables
Environment=PORT=8080
Environment=ENV=production
EnvironmentFile=/etc/myapp/env

# The actual command
ExecStart=/opt/myapp/venv/bin/python app.py
ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/bin/kill -TERM $MAINPID

# Restart behavior
Restart=on-failure
RestartSec=5s
StartLimitBurst=3
StartLimitIntervalSec=60s

# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=myapp

# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/data/myapp

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable --now myapp.service
sudo systemctl status myapp.service

Type=simple    # PID 1 is the main process (default)
Type=forking   # Process daemonizes (forks and parent exits)
Type=notify    # Process sends sd_notify() when ready
Type=oneshot   # Process exits after completion (like a script)
Type=idle      # Like simple, but waits for jobs to finish
Type=exec      # Like simple, systemd waits for exec() to complete

# List all targets
systemctl list-units --type=target

# Show current default target
systemctl get-default

# Change default target
sudo systemctl set-default multi-user.target    # Server (no GUI)
sudo systemctl set-default graphical.target     # Desktop

# Switch target on running system
sudo systemctl isolate rescue.target            # Single-user mode

sudo tee /etc/systemd/system/myapp.target << 'EOF'
[Unit]
Description=My Application Stack
Requires=network.target postgresql.service redis.service
After=network.target postgresql.service redis.service

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable myapp.target

[Unit]
# Hard dependencies
Requires=postgresql.service    # If this fails, this unit also fails
Wants=redis.service            # Soft dependency—optional, won't fail if missing

# Ordering (independent of dependencies)
After=network.target           # Start after these units
Before=nginx.service           # Start before this unit

# Conflict dependencies
Conflicts=maintenance.service  # Cannot run alongside this unit

# Conditional dependencies
BindsTo=container.service      # If container stops, this stops too
PartOf=app-stack.target        # Stop/restart propagates from parent

# Install systemd-graph (part of systemd)
sudo apt install systemd   # Usually pre-installed

# Generate dependency graph for a unit
systemd-analyze dot nginx.service | dot -Tsvg > /tmp/nginx-deps.svg

# Show critical chain (boot performance)
systemd-analyze critical-chain

# Show blame (time each unit took to start)
systemd-analyze blame | head -20

# Total boot time
systemd-analyze time

# Create a drop-in override (preferred method)
sudo systemctl edit nginx.service

# This creates /etc/systemd/system/nginx.service.d/override.conf
# Add only what you want to change:

[Service]
# Override restart policy
Restart=always
RestartSec=2s
# Add an environment variable
Environment=CUSTOM_VAR=value

# View the merged result
systemctl cat nginx.service

# Create socket unit
sudo tee /etc/systemd/system/myapp.socket << 'EOF'
[Unit]
Description=My Application Socket
PartOf=myapp.service

[Socket]
ListenStream=8080
# Accept=yes means a new service instance per connection
# Accept=no means pass fd to one service instance
Accept=no

[Install]
WantedBy=sockets.target
EOF

sudo tee /etc/systemd/system/myapp.service << 'EOF'
[Unit]
Description=My Application
Requires=myapp.socket

[Service]
Type=simple
User=myapp
ExecStart=/opt/myapp/bin/app
# App reads socket from systemd via sd_listen_fds()
StandardInput=socket
EOF

# Enable only the socket—service starts on demand
sudo systemctl enable --now myapp.socket
sudo systemctl status myapp.socket   # Listening but not running
# Connect to port 8080 to trigger the service

# Set resource limits in unit file
sudo systemctl edit myapp.service

[Service]
# CPU: limit to 50% of one CPU core
CPUQuota=50%

# Memory: hard limit
MemoryMax=512M
MemoryHigh=400M   # Soft limit (triggers reclaim)

# I/O weight (100 = default)
IOWeight=50

# Open file limit
LimitNOFILE=65536

# Task limit (threads + processes)
TasksMax=128

# Check current resource usage
systemctl status myapp.service   # Shows cgroup info
sudo systemd-cgtop               # Real-time cgroup resource monitor

# Set limit on running service (temporary)
sudo systemctl set-property myapp.service MemoryMax=256M

# Check cgroup limits
cat /sys/fs/cgroup/system.slice/myapp.service/memory.max

# Create a service to run
sudo tee /etc/systemd/system/cleanup-logs.service << 'EOF'
[Unit]
Description=Clean up old log files

[Service]
Type=oneshot
ExecStart=/usr/local/bin/cleanup-logs.sh
EOF

# Create the timer
sudo tee /etc/systemd/system/cleanup-logs.timer << 'EOF'
[Unit]
Description=Run log cleanup daily
Requires=cleanup-logs.service

[Timer]
# Run daily at 3 AM
OnCalendar=*-*-* 03:00:00
# Run on boot if missed (e.g., server was off)
Persistent=true
# Randomize start time within 10 minutes to avoid thundering herd
RandomizedDelaySec=600

[Install]
WantedBy=timers.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now cleanup-logs.timer

# Check timer schedule
systemctl list-timers
sudo systemctl status cleanup-logs.timer

# View all logs from a service
journalctl -u nginx.service

# Follow in real time
journalctl -u myapp.service -f

# Logs since last boot
journalctl -b

# Logs from previous boot
journalctl -b -1

# Filter by priority (0=emerg to 7=debug)
journalctl -p err -b

# Filter by time
journalctl --since "2026-04-01 10:00:00" --until "2026-04-01 11:00:00"

# JSON output for log processing
journalctl -u myapp.service -o json-pretty | head -50

# Disk usage
journalctl --disk-usage

sudo nano /etc/systemd/journald.conf

[Journal]
Storage=persistent
Compress=yes
MaxRetentionSec=3month
SystemMaxUse=500M
SystemKeepFree=100M

sudo systemctl restart systemd-journald

sudo systemctl status myapp.service
sudo journalctl -u myapp.service -n 50 --no-pager
# Check if ExecStart binary exists and is executable
ls -la /opt/myapp/bin/app
# Check permissions as the service user
sudo -u myapp /opt/myapp/bin/app

# Always reload after editing unit files
sudo systemctl daemon-reload
sudo systemctl restart myapp.service
# Verify the config was loaded
systemctl cat myapp.service

systemd-analyze verify myapp.service
systemd-analyze dot myapp.service | grep -v "\-\->" | head -20

# Add proper ordering
# In [Unit] section:
# After=network-online.target
# Wants=network-online.target
# Note: network.target != network-online.target (online = configured)

systemctl list-timers --all
journalctl -u cleanup-logs.timer
# Check if service unit exists
systemctl status cleanup-logs.service