inotify_instance: add commandline arguments for command and user parsing

on systems with many short lived processes, the cardinality of the
rendered metrics can become quite bad.

this change adds commandline arguments to the script to support
filtering of processes based on the process command or user. the
script logic remains backwards compatible as it continues to
emit metrics for all accessible process information if not filters
are provided.

Signed-off-by: Gordon Bleux <[email protected]>

Author: UiP9AV6Y

Diff for: .gitignore

@@ -0,0 +1,43 @@
+__pypackages__/
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+htmlcov/
+cover/
+target/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+*.manifest
+*.spec
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.pybuilder/
+.env
+.venv
+

Diff for: inotify-instances

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+
+"""
+Expose Linux inotify(7) instance resource consumption.
+
+Operational properties:
+
+  - This script may be invoked as an unprivileged user; in this case, metrics
+    will only be exposed for processes owned by that unprivileged user.
+
+  - No metrics will be exposed for processes that do not hold any inotify fds.
+
+Requires Python 3.5 or later.
+"""
+
+import argparse
+import collections
+import os
+import re
+import sys
+
+from prometheus_client import CollectorRegistry, Gauge, generate_latest
+
+__doc__ = "Exponse Linux Kernel inode information as Prometheus metrics."
+_NotifyTarget = "anon_inode:inotify"
+_Proc = "/proc"
+_CommFile = "cmdline"
+_StatsFile = "status"
+_DescriptorDir = "fd"
+
+
+class Error(Exception):
+    pass
+
+
+class _PIDGoneError(Error):
+    pass
+
+
+_Process = collections.namedtuple(
+    "Process", ["pid", "uid", "command", "inotify_instances"]
+)
+
+
+def _read_bytes(name):
+    with open(name, mode="rb") as f:
+        return f.read()
+
+
+def _pids(proc=_Proc):
+    for n in os.listdir(proc):
+        if not n.isdigit():
+            continue
+        yield int(n)
+
+
+def _pid_uid(status):
+    try:
+        s = os.stat(status)
+    except FileNotFoundError:
+        raise _PIDGoneError()
+    return s.st_uid
+
+
+def _pid_command(cmdline):
+    # Avoid GNU ps(1) for it truncates comm.
+    # https://bugs.launchpad.net/ubuntu/+source/procps/+bug/295876/comments/3
+    try:
+        cmdline = _read_bytes(cmdline)
+    except FileNotFoundError:
+        raise _PIDGoneError()
+
+    if not len(cmdline):
+        return "<zombie>"
+
+    try:
+        prog = cmdline[0 : cmdline.index(0x00)]
+    except ValueError:
+        prog = cmdline
+    return os.path.basename(prog).decode(encoding="ascii", errors="surrogateescape")
+
+
+def _pid_inotify_instances(descriptors):
+    instances = 0
+    try:
+        for fd in os.listdir(descriptors):
+            try:
+                target = os.readlink(os.path.join(descriptors, fd))
+            except FileNotFoundError:
+                continue
+            if target == _NotifyTarget:
+                instances += 1
+    except FileNotFoundError:
+        raise _PIDGoneError()
+    return instances
+
+
+def _get_processes(proc=_Proc):
+    for p in _pids(proc):
+        try:
+            pid = str(p)
+            status = os.path.join(proc, pid, _StatsFile)
+            cmdline = os.path.join(proc, pid, _CommFile)
+            descriptors = os.path.join(proc, pid, _DescriptorDir)
+            yield _Process(
+                p,
+                _pid_uid(status),
+                _pid_command(cmdline),
+                _pid_inotify_instances(descriptors),
+            )
+        except (PermissionError, _PIDGoneError):
+            continue
+
+
+def _generate_metrics(command=None, user=-1, proc=_Proc):
+    registry = CollectorRegistry()
+    namespace = "inotify"
+
+    g = Gauge(
+        "instances",
+        "Total number of inotify instances held open by a process.",
+        ["pid", "uid", "command"],
+        namespace=namespace,
+        registry=registry,
+    )
+
+    for proc in _get_processes(proc):
+        if proc.inotify_instances <= 0:
+            continue
+        elif user >= 0 and user != proc.uid:
+            continue
+        elif command and not command.match(proc.command):
+            continue
+        else:
+            g.labels(proc.pid, proc.uid, proc.command).set(proc.inotify_instances)
+
+    return generate_latest(registry).decode()
+
+
+def main(argv=[__name__]):
+    parser = argparse.ArgumentParser(
+        prog=os.path.basename(argv[0]),
+        exit_on_error=False,
+        description=__doc__,
+    )
+    parser.add_argument(
+        "-c",
+        "--command",
+        default=".+",
+        type=re.compile,
+        dest="command",
+        metavar="REGEX",
+        help="Filter metrics based on the process command",
+    )
+    parser.add_argument(
+        "-u",
+        "--user",
+        default=-1,
+        type=int,
+        dest="user",
+        metavar="UID",
+        help="Filter metrics based on the process user",
+    )
+
+    try:
+        args = parser.parse_args(argv[1:])
+    except argparse.ArgumentError as err:
+        print(err, file=sys.stderr)
+        return 1
+
+    print(_generate_metrics(args.command, args.user, _Proc), end="")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

Diff for: inotify_instances.py

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1/cmdline

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1/fd/0

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/1

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/12

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/123

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/324

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/5

@@ -0,0 +1 @@
+anon_inode:inotify

Diff for: mock/fixtures/inotify_instances/1/fd/82

@@ -0,0 +1,2 @@
+Uid:	0	0	0	0
+Gid:	0	0	0	0

Diff for: mock/fixtures/inotify_instances/1/fd/90

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1/status

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1016/cmdline

@@ -0,0 +1,2 @@
+Uid:	0	0	0	0
+Gid:	0	0	0	0

Diff for: mock/fixtures/inotify_instances/1016/fd/0

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1016/fd/1

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1016/status

@@ -0,0 +1,2 @@
+Uid:	0	0	0	0
+Gid:	0	0	0	0

Diff for: mock/fixtures/inotify_instances/1017/cmdline

@@ -0,0 +1 @@
+/usr/local/bin/rsyslogd

Diff for: mock/fixtures/inotify_instances/1017/fd/0

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1017/fd/1

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1017/status

@@ -0,0 +1,2 @@
+Uid:	0	0	0	0
+Gid:	0	0	0	0

Diff for: mock/fixtures/inotify_instances/1024/cmdline

@@ -0,0 +1 @@
+/dev/null

Diff for: mock/fixtures/inotify_instances/1024/fd/0

@@ -0,0 +1 @@
+/dev/null