{
  "slug": "systemd-journald",
  "url": "/process/systemd-journald/",
  "name": "systemd-journald",
  "kind": "system-daemon",
  "package": "systemd",
  "binary": "/lib/systemd/systemd-journald (Ubuntu 22.04; /usr/lib/systemd/systemd-journald on merged-/usr systems), started by PID 1",
  "config": [
    "/etc/systemd/journald.conf",
    "/etc/systemd/journald.conf.d/*.conf (drop-ins win)"
  ],
  "summary": "The systemd journal daemon: collects kernel, service and syslog messages into binary journal files under /run/log/journal (volatile) or /var/log/journal (persistent), rate-limits chatty services and enforces disk caps.",
  "symptoms": [
    "/var/log/journal in the hundreds of MB or several GB: persistent storage with the default cap (10% of the filesystem, at most 4 GB) and no time-based retention",
    "systemd-journald at steady CPU or I/O: one service is logging thousands of lines a second; 'Suppressed N messages' in journald's own log means the rate limit is already dropping them",
    "journalctl complains about files that are corrupted or not cleanly shut down after a crash or a kill -9: expected, verify and rotate",
    "logs also fill /var/log/syslog: rsyslog is installed as well and journald forwards to it, so every line is stored twice"
  ],
  "safe_to_kill": "restart-instead",
  "safe_to_kill_note": "systemd restarts it immediately, so killing it buys nothing, and a kill -9 leaves journal files marked unclean. systemctl restart systemd-journald is the clean way; services keep writing to the journal socket while it restarts and nothing is lost. The size and CPU problems are solved with configuration and vacuuming, not with signals.",
  "diagnose": [
    {
      "cmd": "journalctl --disk-usage",
      "why": "total size of archived and active journals, the number to compare with the cap"
    },
    {
      "cmd": "journalctl -u systemd-journald -n 5",
      "why": "journald reports its own limits on start: 'System Journal (...) is 832.0M, max 4.0G, 3.1G free' tells you the effective SystemMaxUse"
    },
    {
      "cmd": "journalctl --since '1 hour ago' -o json --output-fields=_SYSTEMD_UNIT | grep -oE '\"_SYSTEMD_UNIT\":\"[^\"]+\"' | sort | uniq -c | sort -rn | head",
      "why": "which units wrote the most lines in the last hour: the CPU and disk you are looking for belong to the top entry, not to journald"
    },
    {
      "cmd": "journalctl -u systemd-journald --since '7 days ago' | grep -ciE 'suppressed|missed'",
      "why": "how often the rate limit (10,000 messages per 30 s per service by default) kicked in; non-zero means a service is flooding and you are losing its messages"
    },
    {
      "cmd": "grep -E '^#?(Storage|SystemMaxUse|SystemKeepFree|MaxRetentionSec|RateLimitIntervalSec|RateLimitBurst|ForwardToSyslog)=' /etc/systemd/journald.conf; ls /etc/systemd/journald.conf.d 2>/dev/null",
      "why": "a leading # means the compiled-in default applies; drop-ins in journald.conf.d override the main file"
    },
    {
      "cmd": "journalctl --verify",
      "why": "checks every journal file; FAIL lines name files to rotate out after a crash"
    },
    {
      "cmd": "ps -o pid,user,rss,etime,args -C systemd-journald",
      "why": "one process owned by root with a small RSS is normal; the journal data lives on disk, not in the daemon"
    }
  ],
  "fixes": [
    {
      "title": "Cap the journal and give it a retention period.",
      "cmd": "sudo mkdir -p /etc/systemd/journald.conf.d && printf '[Journal]\\nSystemMaxUse=500M\\nMaxRetentionSec=1month\\n' | sudo tee /etc/systemd/journald.conf.d/size.conf && sudo systemctl restart systemd-journald",
      "note": "SystemMaxUse defaults to 10% of the filesystem (capped at 4G) and MaxRetentionSec to 0, which disables time-based deletion; journald uses the smaller of SystemMaxUse and SystemKeepFree (15%). A drop-in survives package upgrades, editing journald.conf does not."
    },
    {
      "title": "Reclaim space now.",
      "cmd": "sudo journalctl --vacuum-size=500M",
      "note": "--vacuum-time=30d and --vacuum-files=N are the other two forms. Vacuuming removes archived files only; the active one is rotated first with journalctl --rotate if you need the space immediately."
    },
    {
      "title": "Fix the flooding service, or let the rate limit do its job.",
      "note": "RateLimitIntervalSec=30s and RateLimitBurst=10000 are per service: a unit that exceeds them has the rest of its messages dropped for the interval, with one 'Suppressed' line to say so. Raising the burst hides the problem on disk; lowering the service's log level removes it."
    },
    {
      "title": "Small VM, no need for history across reboots: keep the journal in RAM.",
      "note": "Storage=volatile writes only to /run/log/journal (tmpfs, capped by RuntimeMaxUse); Storage=auto, the usual packaged default, is persistent whenever /var/log/journal exists. Delete that directory and restart journald to switch without editing the config."
    }
  ],
  "harden": [],
  "impostors": [
    {
      "cmd": "readlink /proc/$(pgrep -x systemd-journald | head -1)/exe; ps -o ppid= -p $(pgrep -x systemd-journald | head -1)",
      "why": "the executable is /lib/systemd/systemd-journald or /usr/lib/systemd/systemd-journald and the parent is PID 1; anything else named systemd-journald is borrowing the name"
    }
  ],
  "listens": [],
  "distros": {
    "ubuntu": "22.04 ships /var/log/journal, so storage is persistent, and installs rsyslog with ForwardToSyslog on, so /var/log/syslog duplicates the journal; check dpkg -l rsyslog before blaming journald for disk use. Journal files are group adm: add yourself to adm to read system logs without sudo.",
    "any": "Defaults in this card are the upstream systemd ones from journald.conf(5); distributions may compile in different values, which journald prints on start (see the second diagnose command)."
  },
  "applies_to": "systemd 2xx; options and defaults checked against journald.conf(5) upstream on 2026-10-20 and against systemd 249 on Ubuntu 22.04.",
  "see_also": [
    "unattended-upgrades",
    "kswapd0"
  ],
  "sources": [
    {
      "title": "journald.conf(5) — systemd manual (Storage, SystemMaxUse, SystemKeepFree, MaxRetentionSec, RateLimit*)",
      "url": "https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html",
      "fetched": "2026-10-20T00:00:00.000Z"
    }
  ],
  "tested_on": "Ubuntu 22.04 · systemd 249 (WSL2 with systemd; vacuum and rotate taken from journalctl --help, not run)",
  "updated": "2026-10-20T00:00:00.000Z",
  "changelog": [
    {
      "date": "2026-10-20T00:00:00.000Z",
      "note": "first version"
    }
  ]
}