conmonrs/
oom_watcher.rs

1//! Watching the cgroup of a container process for OOM kills, through the
2//! memory events of the unified hierarchy of cgroup v2. A host with cgroup
3//! v1, or with both in a hybrid setup, runs its containers without the
4//! detection.
5
6mod cgroup_path;
7#[cfg(any(test, feature = "fuzzing"))]
8pub(crate) mod fuzzing;
9mod inotify;
10mod v2;
11
12use anyhow::Context;
13use nix::{
14    sys::{
15        signal::Signal,
16        statfs::{CGROUP2_SUPER_MAGIC, statfs},
17    },
18    unistd::Pid,
19};
20use std::{
21    num::ParseIntError,
22    path::PathBuf,
23    sync::{
24        LazyLock,
25        atomic::{AtomicBool, Ordering},
26    },
27};
28use tokio::fs::OpenOptions;
29use tokio_util::sync::CancellationToken;
30use tracing::{debug, error, warn};
31
32const CGROUP_ROOT: &str = "/sys/fs/cgroup";
33
34/// Capacity of the channel delivering file watch events to one OOM watcher.
35const WATCH_EVENT_CHANNEL_SIZE: usize = 32;
36
37static IS_CGROUP_V2: LazyLock<bool> = LazyLock::new(|| {
38    statfs(CGROUP_ROOT).is_ok_and(|sts| sts.filesystem_type() == CGROUP2_SUPER_MAGIC)
39});
40
41/// Whether the server told that it does not watch for OOM kills, see
42/// [`OOMWatcher::supported`].
43static UNSUPPORTED_WARNED: AtomicBool = AtomicBool::new(false);
44
45/// Watches the cgroup of a process for OOM kills, see
46/// [`OOMWatcher::watch`].
47pub(crate) struct OOMWatcher;
48
49/// What watching a process for OOM kills found.
50#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
51pub(crate) struct OOMEvent {
52    pub(crate) oom: bool,
53
54    /// The cgroup of the process has no memory events, because it got
55    /// removed or has no memory controller, which hides whether an OOM
56    /// happened, but there is a hint which may also stem from a sibling: a
57    /// new OOM kill of an outer cgroup, or none to tell. So it only counts
58    /// for a process which got killed by SIGKILL, see [`OOMEvent::is_oom`].
59    pub(crate) possible_oom: bool,
60}
61
62impl OOMEvent {
63    fn new(oom: bool) -> Self {
64        Self {
65            oom,
66            ..Self::default()
67        }
68    }
69
70    /// A possible OOM if `possible`, none otherwise.
71    pub(crate) fn possible(possible: bool) -> Self {
72        Self {
73            oom: false,
74            possible_oom: possible,
75        }
76    }
77
78    /// Whether the process which exited with `exit_code` got OOM killed.
79    pub(crate) fn is_oom(self, exit_code: i32) -> bool {
80        self.oom || (self.possible_oom && exit_code == 128 + Signal::SIGKILL as i32)
81    }
82}
83
84impl OOMWatcher {
85    /// Watch the cgroup of the process for an OOM until `token` gets
86    /// cancelled, which the exit of the process does, and return what was
87    /// found. The OOM files get written for an OOM found on its own. None if
88    /// the cgroup could not be watched, like without cgroup v2, or the
89    /// process was gone already.
90    pub(crate) async fn watch(
91        token: &CancellationToken,
92        pid: Pid,
93        exit_paths: &[PathBuf],
94    ) -> Option<OOMEvent> {
95        if !Self::supported(*IS_CGROUP_V2, &UNSUPPORTED_WARNED) {
96            return None;
97        }
98        let event = Self::oom_handling_cgroup_v2(token, pid)
99            .await
100            .context("set up cgroup v2 OOM handling")
101            .inspect_err(|e| error!("Unable to watch OOM: {e:#}"))
102            .ok()
103            .flatten();
104        if event.is_some_and(|event| event.oom) {
105            Self::write_oom_files(exit_paths).await;
106        }
107        debug!("Done watching for OOMs");
108        event
109    }
110
111    /// Whether the OOM kills of the host can be watched, which takes the
112    /// unified hierarchy of cgroup v2 at the cgroup root. Without it, like
113    /// with cgroup v1 or a hybrid setup, the containers run without the OOM
114    /// detection, which a warning tells once per server, as noted in
115    /// `warned`, instead of with every container.
116    fn supported(cgroup_v2: bool, warned: &AtomicBool) -> bool {
117        if !cgroup_v2 && !warned.swap(true, Ordering::Relaxed) {
118            warn!(
119                "No cgroup v2 at {CGROUP_ROOT}, the OOM kills of containers do not get detected: cgroup v1 and hybrid setups are not supported"
120            );
121        }
122        cgroup_v2
123    }
124
125    /// The counters of a cgroup file of `key value` lines, like
126    /// `memory.events`, by their key.
127    fn counters(content: &str) -> impl Iterator<Item = (&str, Result<u64, ParseIntError>)> {
128        content
129            .lines()
130            .filter_map(|line| line.split_once(' '))
131            .map(|(key, value)| (key, value.trim().parse()))
132    }
133
134    /// Create the empty OOM files, like conmon does. A failure is only
135    /// logged, the exit gets reported either way, only without the OOM. A
136    /// symlink at the path does not get followed, like for the log files.
137    pub(crate) async fn write_oom_files(exit_paths: &[PathBuf]) {
138        for path in exit_paths {
139            debug!("Writing OOM file: {}", path.display());
140            let created = OpenOptions::new()
141                .write(true)
142                .create(true)
143                .truncate(true)
144                .custom_flags(libc::O_NOFOLLOW)
145                .open(path)
146                .await;
147            if let Err(e) = created {
148                error!("Unable to create OOM file {}: {e:#}", path.display());
149            }
150        }
151    }
152}
153
154#[cfg(test)]
155mod tests {
156    use super::*;
157    use crate::test_util::Log;
158
159    #[test]
160    fn only_cgroup_v2_gets_watched() -> anyhow::Result<()> {
161        let (log, _guard) = Log::capture();
162        let warned = AtomicBool::new(false);
163        assert!(OOMWatcher::supported(true, &warned));
164        assert!(!warned.load(Ordering::Relaxed));
165
166        // Every container asks, the first one warns.
167        for _ in 0..3 {
168            assert!(!OOMWatcher::supported(false, &warned));
169        }
170        assert!(warned.load(Ordering::Relaxed));
171        assert!(OOMWatcher::supported(true, &warned));
172        let log = log.contents()?;
173        assert_eq!(log.matches("WARN").count(), 1, "{log}");
174        assert_eq!(log.matches("No cgroup v2 at").count(), 1, "{log}");
175        Ok(())
176    }
177
178    #[test]
179    fn oom_event_needs_kill_for_ancestor_oom() {
180        assert!(OOMEvent::new(true).is_oom(0));
181        assert!(!OOMEvent::default().is_oom(137));
182        let ancestor = OOMEvent {
183            oom: false,
184            possible_oom: true,
185        };
186        assert!(ancestor.is_oom(137));
187        assert!(!ancestor.is_oom(143));
188        assert!(!ancestor.is_oom(1));
189    }
190
191    #[tokio::test]
192    async fn oom_files_do_not_follow_symlinks() -> anyhow::Result<()> {
193        let dir = tempfile::tempdir()?;
194        let target = dir.path().join("target");
195        std::fs::write(&target, "untouched")?;
196        let link = dir.path().join("oom");
197        std::os::unix::fs::symlink(&target, &link)?;
198        let plain = dir.path().join("plain");
199
200        // The failure for the symlink does not keep the other file from
201        // getting created.
202        OOMWatcher::write_oom_files(&[link.clone(), plain.clone()]).await;
203        assert_eq!(std::fs::read_to_string(&target)?, "untouched");
204        assert!(std::fs::symlink_metadata(&link)?.is_symlink());
205        assert!(plain.is_file());
206        Ok(())
207    }
208}
conmonrs/oom_watcher.rs

conmonrs/
oom_watcher.rs
