Linux Performance Monitoring

# Load average
uptime
# 10:30:05 up 5 days, 2:15, 2 users, load average: 1.25, 0.87, 0.64
#                                                    1min  5min  15min
# Rule: load average / number of CPUs = saturation
# 1.25 load on a 4-core system = 31% saturated (healthy)
# 8.0  load on a 4-core system = 200% saturated (heavy queue)

nproc                    # number of logical CPU cores
lscpu                    # full CPU info (cores, threads, NUMA)

# vmstat — virtual memory statistics
vmstat 1 5               # 5 samples, 1 second apart
# procs  memory          swap  io    system     cpu
# r  b   swpd   free      si  so    bi  bo      in  cs   us sy id wa st
# 2  0   0    512000     0    0     100 200   1500 2000  40 10 45 5  0
# r: runnable (queued) processes — if consistently > nproc, CPU saturated
# b: uninterruptible sleep (waiting for I/O) — if high, I/O bottleneck
# wa (cpu): % time waiting for I/O — high wa = disk/network bottleneck
# cs: context switches per second — very high = too many threads/processes
# us: user CPU, sy: kernel CPU, id: idle

# CPU profiling with perf
perf top                 # real-time CPU profiler (which functions are hot)
perf record -g ./myapp   # record call graph
perf report              # analyze recorded data

# free — memory overview
free -h
#               total   used    free    shared  buff/cache  available
# Mem:          15.6G   4.2G    2.1G    512M    9.3G        10.8G
#                                                            ↑ THIS is the usable number
# "available" = free + reclaimable cache — what the OS can give a new process

# /proc/meminfo — detailed breakdown
cat /proc/meminfo
# MemTotal:       16334332 kB
# MemFree:         2150000 kB
# MemAvailable:   11090000 kB  ← reliable "usable" metric
# Buffers:          450000 kB  ← disk block buffer
# Cached:          8900000 kB  ← page cache (can be reclaimed)
# SwapTotal:       2097152 kB
# SwapFree:        2097000 kB  ← if swap is in use, you have a memory problem

# Is system swapping?
vmstat 1 | awk '{print $7, $8}'   # si (swap in), so (swap out)
# Non-zero swap out = memory pressure = performance impact

# What is using memory?
ps aux --sort=-%mem | head -10     # top 10 processes by memory
smem -r -s rss | head             # smem: accounts for shared memory correctly

# OOM killer history (did Linux kill a process for OOM?)
dmesg | grep -i "killed process"
journalctl -k | grep -i oom

# iostat — I/O statistics
iostat -x 1 3            # extended stats, 1s interval, 3 samples
# Device   r/s   w/s  rkB/s  wkB/s  await  util%
# sda      5.2  42.0   82.4  920.0   8.2    45.0
# ↑ sda: 45% utilised — moderate
# await: average time (ms) for I/O request — high = I/O bottleneck
# util%: device busy % — 100% = saturated = bottleneck

# sda: SSD typically saturates at util% 100% AND high await
# For IOPS-intensive workloads: await > 5ms on SSD = investigate

# iotop — which process is doing I/O (like top for disk)
iotop -o              # -o: only show processes doing I/O
# TID  PRIO USER  DISK READ  DISK WRITE  COMMAND
# 1234  be/4 mysql   0.00 B/s   5.32 M/s  mysqld

# df — disk space usage
df -h                 # human-readable, all mounted filesystems
df -i                 # inode usage (exhausted inodes = "no space" even with space)

# du — directory size
du -sh /var/log/      # size of directory
du -sh /* 2>/dev/null | sort -rh | head -10  # largest top-level dirs

# Find large files (>100 MB)
find / -type f -size +100M -exec ls -lh {} \; 2>/dev/null | sort -k5 -rh

# Open files by process
lsof -p 1234          # files open by PID 1234
lsof /var/log/nginx/  # which processes have files open in this dir