Expose node exporter for scraping

bundles/uberspace-clerie-monitoring/files/prometheus-node-exporter-web-config.yml

@@ -0,0 +1,2 @@
+basic_auth_users:
+  monitor: "$2y$05$b4TibqWAj3MDv3g.SCaPZe9r6jV8Vowvn/CBZOdScwgkE7JcIuUfK"

bundles/uberspace-clerie-monitoring/files/prometheus-node-exporter.ini

@@ -0,0 +1,9 @@
+[program:prometheus-node-exporter]
+command=prometheus-node-exporter
+    --web.listen-address 0.0.0.0:9100
+    --web.config.file=/home/{{ node.username }}/.config/prometheus-node-exporter/web-config.yml
+    --collector.disable-defaults
+    --collector.textfile
+    --collector.textfile.directory=/home/{{ node.username }}/.local/state/prometheus-node-exporter/textfiles
+    --collector.time
+startsecs=60

bundles/uberspace-clerie-monitoring/items.py

@@ -0,0 +1,59 @@
+uberspaceify = repo.libs.uberspace.Uberspaceify(node)
+
+directories = uberspaceify.directories({
+    f'/home/{node.username}/.local/state/prometheus-node-exporter/textfiles': {
+    },
+})
+
+files = uberspaceify.files({
+    f'/home/{node.username}/.bwdownloads/prometheus-node-exporter.tar.gz': {
+        "content_type": "download",
+        "source": "https://github.com/prometheus/node_exporter/releases/download/v1.9.0/node_exporter-1.9.0.linux-amd64.tar.gz",
+        "content_hash": "847bd800f3cae9c8e613c9d2fb59ffc7bb3eb764",
+        "triggers": [
+            "action:prometheus-node-exporter-install",
+        ],
+    },
+    f"/home/{node.username}/etc/services.d/prometheus-node-exporter.ini": {
+        "source": "prometheus-node-exporter.ini",
+        "content_type": "jinja2",
+        "triggers": [
+            "action:supervisord-reread",
+            "svc_uberspace_supervisord:prometheus-node-exporter:update",
+        ],
+    },
+    f"/home/{node.username}/.config/prometheus-node-exporter/web-config.yml": {
+        "source": "prometheus-node-exporter-web-config.yml",
+        "triggers": [
+            "svc_uberspace_supervisord:prometheus-node-exporter:restart",
+        ],
+    },
+})
+
+actions["prometheus-node-exporter-exists"] = {
+    "command": 'true',
+    "unless": f'[[ -f "/home/{node.username}/bin/prometheus-node-exporter" && -x "/home/{node.username}/bin/prometheus-node-exporter" ]]',
+    "triggers": [
+        "action:prometheus-node-exporter-install",
+    ],
+}
+
+actions["prometheus-node-exporter-install"] = {
+    "command": f'tar -xf /home/{node.username}/.bwdownloads/prometheus-node-exporter.tar.gz node_exporter*/node_exporter --to-stdout > /home/{node.username}/bin/prometheus-node-exporter && chmod +x /home/{node.username}/bin/prometheus-node-exporter',
+    "triggered": True,
+    "needs": [
+        f"file:/home/{node.username}/.bwdownloads/prometheus-node-exporter.tar.gz",
+    ],
+}
+
+svc_uberspace_supervisord["prometheus-node-exporter"] = {
+    "needs": [
+        f"file:/home/{node.username}/etc/services.d/prometheus-node-exporter.ini",
+    ],
+}
+
+uberspace_web_backend["/.node-exporter"] = {
+    "backend": "http",
+    "port": 9100,
+    "remove_prefix": True,
+}

bundles/uberspace-supervisord/items.py

@@ -0,0 +1,18 @@
+uberspaceify = repo.libs.uberspace.Uberspaceify(node)
+
+directories = uberspaceify.directories({
+    f'/home/{node.username}/etc/services.d': {
+        "purge": True,
+        "triggers": [
+            "action:supervisord-reread",
+        ],
+    },
+})
+
+actions["supervisord-reread"] = {
+    "command": "supervisorctl reread",
+    "triggered": True,
+    "needed_by": [
+        "svc_uberspace_supervisord:",
+    ],
+}

items/svc_uberspace_supervisord.py

@@ -0,0 +1,80 @@
+from shlex import quote
+
+from bundlewrap.exceptions import BundleError
+from bundlewrap.items import Item
+
+
+def svc_start(node, svcname):
+    return node.run("supervisorctl start {}".format(quote(svcname)), may_fail=True)
+
+def svc_running(node, svcname):
+    result = node.run("supervisorctl status {}".format(quote(svcname)), may_fail=True)
+    return result.return_code == 0
+
+def svc_stop(node, svcname):
+    return node.run("supervisorctl stop {}".format(quote(svcname)), may_fail=True)
+
+
+class SvcSystemd(Item):
+    """
+    A service managed by supervisord in uberspace.
+    """
+    BUNDLE_ATTRIBUTE_NAME = "svc_uberspace_supervisord"
+    ITEM_ATTRIBUTES = {
+        'running': True,
+    }
+    ITEM_TYPE_NAME = "svc_uberspace_supervisord"
+
+    def __repr__(self):
+        return "<SvcUberspaceSupervisord name:{} running:{}>".format(
+            self.name,
+            self.attributes['running'],
+        )
+
+    def cdict(self):
+        cdict = {}
+        for option, value in self.attributes.items():
+            if value is not None:
+                cdict[option] = value
+        return cdict
+
+    def fix(self, status):
+        if 'running' in status.keys_to_fix:
+            if self.attributes['running']:
+                svc_start(self.node, self.name)
+            else:
+                svc_stop(self.node, self.name)
+
+    def get_canned_actions(self):
+        return {
+            'stop': {
+                'command': "supervisorctl stop {}".format(self.name),
+                'needed_by': {self.id},
+            },
+            'restart': {
+                'command': "supervisorctl restart {}".format(self.name),
+                'needs': {self.id},
+            },
+            'update': {
+                'command': "supervisorctl update {}".format(self.name),
+                'needs': {self.id},
+            },
+        }
+
+    def sdict(self):
+        return {
+            'running': svc_running(self.node, self.name),
+        }
+
+    @classmethod
+    def validate_attributes(cls, bundle, item_id, attributes):
+        for attribute in ('running'):
+            if attributes.get(attribute, None) not in (True, False, None):
+                raise BundleError(_(
+                    "expected boolean or None for '{attribute}' on {item} in bundle '{bundle}'"
+                ).format(
+                    attribute=attribute,
+                    bundle=bundle.name,
+                    item=item_id,
+                ))
+

items/uberspace_web_backend.py

@@ -0,0 +1,138 @@
+from bundlewrap.exceptions import BundleError
+from bundlewrap.items import Item
+
+
+def backend_segments_to_dict(segments):
+    attributes = {}
+
+    name = segments[0].strip()
+
+    attributes["backend"] = segments[1].strip().rstrip(",")
+
+    attributes["port"] = None
+
+    if ":" in attributes["backend"]:
+        backend_segments = attributes["backend"].split(":")
+        attributes["backend"] = backend_segments[0]
+        attributes["port"] = int(backend_segments[1])
+
+    attributes["remove_prefix"] = None
+
+    flag = segments[2]
+    if flag == "--remove-prefix":
+        attributes["remove_prefix"] = True
+
+    return attributes
+
+def get_backend_config(node, name):
+    result = node.run("uberspace web backend list", may_fail=True)
+    backend_lines = result.stdout_text.splitlines()
+
+    for line in backend_lines:
+        segments = line.split("=>", 1)[0].strip().split(" ")
+        if segments[0] == name:
+            d = backend_segments_to_dict(segments)
+            return d
+
+def make_display_string(name, attributes):
+    backend_name = attributes.get("backend")
+    if attributes.get("backend") == "http":
+        backend_name += f":{attributes.get('port')}"
+
+    args = [ backend_name ]
+
+    if attributes["remove_prefix"]:
+        args.append("--remove-prefix")
+
+    arg_string = ", ".join(args)
+
+    return f"{name} {arg_string}"
+
+def make_option_string(name, attributes):
+    out = []
+
+    out.append(name)
+    out.append(f" --{attributes.get('backend')}")
+
+    if attributes.get("backend") == "http":
+        out.append("--port")
+        out.append(str(attributes.get('port')))
+
+    if attributes["remove_prefix"]:
+        out.append("--remove-prefix")
+
+    return " ".join(out)
+
+class UberspaceWebBackend(Item):
+    """
+    Manage uberspace web backends
+    """
+    BUNDLE_ATTRIBUTE_NAME = "uberspace_web_backend"
+    ITEM_ATTRIBUTES = {
+        "backend": "apache",
+        "port": None,
+        "remove_prefix": None,
+    }
+    ITEM_TYPE_NAME = "uberspace_web_backend"
+
+    def __repr__(self):
+        return f"<UberspaceWebBackend {make_display_string(self.name, self.attributes)}>"
+
+    def cdict(self):
+        cdict = {}
+        for option, value in self.attributes.items():
+            if value is not None:
+                cdict[option] = value
+        return cdict
+
+    def fix(self, status):
+        self.node.run(f"uberspace web backend set {make_option_string(self.name, self.attributes)}", may_fail=True)
+
+    def sdict(self):
+        return get_backend_config(self.node, self.name)
+
+    @classmethod
+    def validate_attributes(cls, bundle, item_id, attributes):
+
+        if attributes.get("backend") == "apache":
+            if attributes.get("port") is not None:
+                raise BundleError(_(
+                    "do not set 'port' while using 'apache' for 'backend' on {item} in bundle '{bundle}'"
+                ).format(
+                    item=item_id,
+                    bundle=bundle.name,
+                ))
+
+            if attributes.get("remove_prefix") is not None:
+                raise BundleError(_(
+                    "do not set 'port' while using 'apache' for 'backend' on {item} in bundle '{bundle}'"
+                ).format(
+                    item=item_id,
+                    bundle=bundle.name,
+                ))
+
+        elif attributes.get("backend") == "http":
+            if not isinstance(attributes.get("port"), int):
+                raise BundleError(_(
+                    "'port' required and needs to be numeric while using 'http' for 'backend' on {item} in bundle '{bundle}'"
+                ).format(
+                    item=item_id,
+                    bundle=bundle.name,
+                ))
+
+            if attributes.get("remove_prefix") not in [True, False, None]:
+                raise BundleError(_(
+                    "expected boolean for 'remove_prefix' on {item} in bundle '{bundle}'"
+                ).format(
+                    item=item_id,
+                    bundle=bundle.name,
+                ))
+
+        else:
+            raise BundleError(_(
+                "expected 'apache' or 'http' for 'backend' on {item} in bundle '{bundle}'"
+            ).format(
+                item=item_id,
+                bundle=bundle.name,
+            ))
+

nodes.py

@@ -1,37 +1,48 @@
 bws = libs.bwsops.BwSops("secrets.json")
 uberspaceify = libs.uberspace.Uberspaceify()
 
+def generate_default_repos(node_name, username):
+    return {
+        "main-cyan": {
+            "repo_password": bws.get([node_name, "clerie-backup-job-main"]),
+            "repo_url": f"https://cyan.backup.clerie.de/{node_name}/main",
+            "auth_username": node_name,
+            "auth_password": bws.get([node_name, "clerie-backup-target-cyan"]),
+            "files": [
+                f"/home/{username}",
+                f"/var/www/virtual/{username}",
+            ],
+            "excludes": [
+                "/home/*/.cache",
+            ],
+        },
+        "main-magenta": {
+            "repo_password": bws.get([node_name, "clerie-backup-job-main"]),
+            "repo_url": f"https://magenta.backup.clerie.de/{node_name}/main",
+            "auth_username": node_name,
+            "auth_password": bws.get([node_name, "clerie-backup-target-magenta"]),
+            "files": [
+                f"/home/{username}",
+                f"/var/www/virtual/{username}",
+            ],
+            "excludes": [
+                "/home/*/.cache",
+            ],
+        },
+    }
+
 nodes = uberspaceify.nodes({
     "clerie.uber.space": {
         "bundles": (
             "uberspace-redirect-clerie",
             "uberspace-clerie-backup",
+            "uberspace-clerie-monitoring",
             "uberspace-crontab-dir",
+            "uberspace-supervisord",
         ),
         "metadata": {
             "clerie-backup": {
-                "repos": {
+                "repos": generate_default_repos("clerie.uber.space", "clerie"),
-                    "main-cyan": {
-                        "repo_password": bws.get("clerie.uber.space/clerie-backup/main-cyan/repo_password"),
-                        "repo_url": "https://cyan.backup.clerie.de/clerie.uber.space/main",
-                        "auth_username": "clerie.uber.space",
-                        "auth_password": bws.get("clerie.uber.space/clerie-backup/main-cyan/auth_password"),
-                        "files": [
-                            "/home/clerie",
-                            "/var/www/virtual/clerie",
-                        ],
-                    },
-                    "main-magenta": {
-                        "repo_password": bws.get("clerie.uber.space/clerie-backup/main-magenta/repo_password"),
-                        "repo_url": "https://magenta.backup.clerie.de/clerie.uber.space/main",
-                        "auth_username": "clerie.uber.space",
-                        "auth_password": bws.get("clerie.uber.space/clerie-backup/main-magenta/auth_password"),
-                        "files": [
-                            "/home/clerie",
-                            "/var/www/virtual/clerie",
-                        ],
-                    },
-                },
             },
             "cron": {
                 "jobs": {
@@ -51,28 +62,7 @@ nodes = uberspaceify.nodes({
         ),
         "metadata": {
             "clerie-backup": {
-                "repos": {
+                "repos": generate_default_repos("cleriewi.uber.space", "cleriewi"),
-                    "main-cyan": {
-                        "repo_password": bws.get("cleriewi.uber.space/clerie-backup-job-main"),
-                        "repo_url": "https://cyan.backup.clerie.de/cleriewi.uber.space/main",
-                        "auth_username": "cleriewi.uber.space",
-                        "auth_password": bws.get("cleriewi.uber.space/clerie-backup-target-cyan"),
-                        "files": [
-                            "/home/clerie",
-                            "/var/www/virtual/clerie",
-                        ],
-                    },
-                    "main-magenta": {
-                        "repo_password": bws.get("cleriewi.uber.space/clerie-backup-job-main"),
-                        "repo_url": "https://magenta.backup.clerie.de/cleriewi.uber.space/main",
-                        "auth_username": "cleriewi.uber.space",
-                        "auth_password": bws.get("cleriewi.uber.space/clerie-backup-target-magenta"),
-                        "files": [
-                            "/home/cleriewi",
-                            "/var/www/virtual/cleriewi",
-                        ],
-                    },
-                },
             },
             "cron": {
                 "jobs": {

secrets.json

@@ -1,15 +1,8 @@
 {
 	"clerie.uber.space": {
-		"clerie-backup": {
+		"clerie-backup-job-main": "ENC[AES256_GCM,data:cEKZuviDYpzyhbjydOYPWF2wzozQwutSRCWo0bGECoTxV1R+PTrX4C7rc/Xj0oGCgclEfZaNvUGOfm4h7Y9xjA==,iv:RiQ/+yFgYS8Sd/fh+Z2SmKlTCIMGU3HtkfjUBDuE7sk=,tag:KC6qUKW+0gffpf3SVjUhdQ==,type:str]",
-			"main-cyan": {
+		"clerie-backup-target-cyan": "ENC[AES256_GCM,data:V/Bgfu5Thb3pUDtjk20xUQQXBgnmhv+H9R1bABYX2o3JutLntexO4LGeECZdsTIrKEVX/mwcXpFmhesYjTWQqw==,iv:jrxsku6VDCjhzLbhqa2MdOfvu7p6rKWr4WGYP6GCU6I=,tag:7AFT9WcGDH49c2DSy1ZBcw==,type:str]",
-				"repo_password": "ENC[AES256_GCM,data:BjyMOOGFdcT6e/zuQKPW88XiKhL26KYKebP0dsevMHmKoxQdIw9R/KS/GIlMlJKsz2wwH6nk5adiLBVD9+7TCA==,iv:1/iT4/QwRuZL3pq2D9nYiqTEpMHtf29r1mAYOzkxQeA=,tag:4LxvLRwfJYd9erPhBKb1Bw==,type:str]",
+		"clerie-backup-target-magenta": "ENC[AES256_GCM,data:6E6JJbbnG20FbGtneNZKUok/DsIplcnTEYvJn2zJcZMha1xk9SZGs8Pf4P6odkEjCFN06FeciuS/vgGa7rOP7A==,iv:tTnmJhnHefIusGNIHcZGqDLNEYkUXR0ITsHrDdJl2oo=,tag:LIl7xoiTOs4i2xzVW+N3wg==,type:str]"
-				"auth_password": "ENC[AES256_GCM,data:j6QJeJ7SKKoEnpiO+YxvHfcP38KVop0bfAScRk0mWeBr+8S+3sYMGnPcVbhoHcqJsWyeeEDomhm/GniGza2zLA==,iv:bxL7Hh9jVZ4wc4/69qSmRZZHcCxlEpllIu9zKA4tRy0=,tag:W4cNmbJgDZlqshUN39ieLw==,type:str]"
-			},
-			"main-magenta": {
-				"repo_password": "ENC[AES256_GCM,data:nDPphjFK0gY77Vq+QLhWRhUM+IxgPD+rIgYS99HCDLS5YVLiPK56kizXRWlOPpq/uUr9bdhLnQAJ8FOBbhvBKA==,iv:HE/v/cVfIrjEXdb820pGJe2hqpYaXiIqDTBr0qTg7AY=,tag:yFpn+0rI9mzGv7vdxUBKjQ==,type:str]",
-				"auth_password": "ENC[AES256_GCM,data:I3NbXg6Gk4/7z2Ct7L10/mASKxInAFbKN9s9O5Lz6mTfbL35n+cu58H2xP/Nxqu/AJMCcOoXwNHhA/iLGH24RQ==,iv:TwdUR7jnVYK0PnQmMs5P+TrXEtyCxQXbxWPDsmdEPbc=,tag:Sat2Wlb4vYMoH+VRgKjaFQ==,type:str]"
-			}
-		}
 	},
 	"cleriewi.uber.space": {
 		"clerie-backup-job-main": "ENC[AES256_GCM,data:f6Hl80iUbjV3h69vjAyyrrvnnVKxk2DhuewE/kP0+9AX/yihlD5aOosqHnF8kbh/5Snow7/1k8CR2xpfSbiDaA==,iv:zdNhrrNr4pWUf2MpuXaBzbjkQ8xbfcZhI1/GsjXj8oY=,tag:+KpPOBwAG5uvoDtGYGBE0Q==,type:str]",
@@ -22,8 +15,8 @@
 		"azure_kv": null,
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2025-02-16T18:14:02Z",
+		"lastmodified": "2025-02-16T18:41:27Z",
-		"mac": "ENC[AES256_GCM,data:bS9Q1sEo0ItLJJYFasJsNshfxeCYR2uFF4FVzoaQ6AHMSz76eC/uPU0njoc+tJK6qOYcPPpTGddo6bhv4A7oXaTm+vmfinetXbsvVGi4jX16LP+CEY/uqgqxfQ7+rHB5qdEyi/8Vb7N3WK2166xq3eqsBc3U+djTK2g1UB0Y2Xk=,iv:PFNuQtQENEI07fv/BA7PLth98+sSPMIWsWweQp3kecI=,tag:dW4UY7m2qiZ5Pj3w0V6REA==,type:str]",
+		"mac": "ENC[AES256_GCM,data:QyU1INnlZVP5RNPczuZEAeCah+c3rjWHhiGGjDn87tUSp+OwEkL44Hosr9vThk6FNdKWbtqcUh1wBW/UCgy5/jmh2BHv3pTIOzkXWAD1fy/Kb/jNYo0IH1+7cte98+NcDPw7do4k1fYM/H5VD3SPpGp5bWxEcrkrZuiupThuduI=,iv:QXqL4hbymO7uOBfghYZwSFgTWUnBeA52sHl201ChRME=,tag:c8Za2rcaO5WRnu4HIJtWWQ==,type:str]",
 		"pgp": [
 			{
 				"created_at": "2025-02-15T16:00:02Z",