diff --git a/README.md b/README.md index 39573a3..4b46b3b 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,14 @@ sudo mkdir -p /srv/media/{movies,shows,downloads,immich} sudo chown -R 1000:1000 /srv/media ``` -### 2. Monitoring premissions +### 2. Monitoring permissions ```bash -sudo mkdir -p /srv/homelab/data/monitoring/{prometheus,loki,grafana} +sudo mkdir -p /srv/homelab/data/monitoring/{prometheus,loki,grafana,alloy} sudo chown -R 65534:65534 /srv/homelab/data/monitoring/prometheus sudo chown -R 10001:10001 /srv/homelab/data/monitoring/loki sudo chown -R 472:472 /srv/homelab/data/monitoring/grafana +sudo chown -R 10001:10001 /srv/homelab/data/monitoring/alloy ``` ### 3. Env @@ -56,22 +57,27 @@ docker compose -f stacks/media/compose.yaml down ## Ports -| Service | Port(s) | Stack | -|-----------|-------------|-----------| -| Sonarr | 8989 | media | -| Radarr | 7878 | media | -| Lidarr | 8686 | media | -| Prowlarr | 9696 | media | -| qBittorrent | 8081, 6881 (tcp/udp) | media | -| Jellyfin | 8096 | media | -| Immich | 2283 | media | -| Navidrome | 4533 | media | -| Seer | 5055 | media | -| Grafana | 3034 | monitoring | -| Prometheus| 9094 | monitoring | -| Loki | 3100 | monitoring | -| cAdvisor | 8088 | monitoring | -| Glance | 9090 | monitoring | -| Portainer | 9443, 8000 | monitoring | -| Gitea | 3000, 222 | storage | -| Copyparty | 3923 | storage | +| Service | Port(s) | Stack | +|---------------|----------------------|-----------| +| **Media** | | | +| Sonarr | 8989 | media | +| Radarr | 7878 | media | +| Lidarr | 8686 | media | +| Prowlarr | 9696 | media | +| qBittorrent | 8081, 6881 (tcp/udp) | media | +| Jellyfin | 8096 | media | +| Immich | 2283 | media | +| Navidrome | 4533 | media | +| Seer | 5055 | media | +| **Monitoring**| | | +| Grafana | 3034 | monitoring| +| Prometheus | 9094 | monitoring| +| Loki | 3100 | monitoring| +| Node Exporter | 9100 | monitoring| +| Alloy | 12345 | monitoring| +| cAdvisor | 8088 | monitoring| +| Glance | 9090 | monitoring| +| Portainer | 9443, 8000 | monitoring| +| **Storage** | | | +| Gitea | 3000, 222 | storage | +| Copyparty | 3923 | storage | diff --git a/config/monitoring/alloy.alloy b/config/monitoring/alloy.alloy new file mode 100644 index 0000000..0591e0c --- /dev/null +++ b/config/monitoring/alloy.alloy @@ -0,0 +1,72 @@ +// Log pipeline: files, Docker, journald → Loki (replaces Promtail). +// See: https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/ + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} + +// varlogs: /var/log/**/*.log +loki.source.file "varlogs" { + targets = [ + { __path__ = "/var/log/**/*.log", job = "varlogs", host = "cyberdeck" }, + ] + forward_to = [loki.write.default.receiver] + file_match { + enabled = true + sync_period = "10s" + } +} + +// docker: container logs with docker parsing and container label from path +loki.source.file "docker" { + targets = [ + { __path__ = "/var/lib/docker/containers/*/*-json.log", job = "docker", host = "cyberdeck" }, + ] + forward_to = [loki.process.docker.receiver] + file_match { + enabled = true + sync_period = "10s" + } +} + +loki.process "docker" { + forward_to = [loki.relabel.docker.receiver] + stage.docker {} +} + +loki.relabel "docker" { + forward_to = [loki.write.default.receiver] + rule { + source_labels = ["filename"] + regex = "/var/lib/docker/containers/([^/]+)/.*" + target_label = "container" + replacement = "$1" + } +} + +// journald: systemd journal with unit, host, level labels +loki.relabel "journald" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal__priority_keyword"] + target_label = "level" + } +} + +loki.source.journal "journald" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journald.rules + labels = { job = "journald", host = "cyberdeck" } + path = "/var/log/journal" + max_age = "12h" +} diff --git a/config/monitoring/loki-config.yaml b/config/monitoring/loki-config.yaml index 7aa27f1..7c085bc 100644 --- a/config/monitoring/loki-config.yaml +++ b/config/monitoring/loki-config.yaml @@ -1,49 +1,55 @@ +# Loki 3.6 single-binary config (from official loki-local-config.yaml). +# See: https://grafana.com/docs/loki/latest/configure/ + auth_enabled: false server: http_listen_port: 3100 + http_listen_address: 0.0.0.0 + grpc_listen_port: 9096 log_level: info + grpc_server_max_concurrent_streams: 1000 -ingester: - wal: - dir: /loki/wal - lifecycler: - ring: - kvstore: - store: inmemory - replication_factor: 1 - chunk_idle_period: 5m - chunk_target_size: 1048576 - max_transfer_retries: 0 +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + enable_multi_variant_queries: true schema_config: configs: - from: 2020-10-24 - store: boltdb + store: tsdb object_store: filesystem - schema: v11 + schema: v13 index: prefix: index_ - period: 168h + period: 24h -storage_config: - boltdb: - directory: /loki/index - filesystem: - directory: /loki/chunks +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 -limits_config: - enforce_metric_name: false - reject_old_samples: true - reject_old_samples_max_age: 168h - split_queries_by_interval: 15m +ruler: + alertmanager_url: http://localhost:9093 -querier: - max_outstanding_per_tenant: 4096 - -chunk_store_config: - max_look_back_period: 0s - -table_manager: - retention_deletes_enabled: true - retention_period: 168h +frontend: + encoding: protobuf diff --git a/config/monitoring/promtail-config.yaml b/config/monitoring/promtail-config.yaml deleted file mode 100644 index 764f6b3..0000000 --- a/config/monitoring/promtail-config.yaml +++ /dev/null @@ -1,48 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: - - job_name: varlogs - static_configs: - - targets: [localhost] - labels: - job: varlogs - host: cyberdeck - __path__: /var/log/**/*.log - - - job_name: docker - static_configs: - - targets: [localhost] - labels: - job: docker - host: cyberdeck - __path__: /var/lib/docker/containers/*/*-json.log - pipeline_stages: - - docker: {} - relabel_configs: - - source_labels: [__path__] - regex: "/var/lib/docker/containers/([^/]+)/.*" - target_label: container - replacement: "$1" - - - job_name: journald - journal: - path: /var/log/journal - max_age: 12h - labels: - job: journald - host: cyberdeck - relabel_configs: - - source_labels: [__journal__systemd_unit] - target_label: unit - - source_labels: [__journal__hostname] - target_label: host - - source_labels: [__journal__priority_keyword] - target_label: level diff --git a/stacks/media/immich.yaml b/stacks/media/immich.yaml index 5d95ad4..12928c6 100644 --- a/stacks/media/immich.yaml +++ b/stacks/media/immich.yaml @@ -39,7 +39,7 @@ services: immich-redis: container_name: immich_redis - image: docker.io/valkey/valkey:9@sha256:546304417feac0874c3dd576e0952c6bb8f06bb4093ea0c9ca303c73cf458f63 + image: docker.io/valkey/valkey:9 healthcheck: test: redis-cli ping || exit 1 restart: unless-stopped @@ -51,7 +51,7 @@ services: immich-postgres: container_name: immich_postgres - image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23 + image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 env_file: - ./immich.env shm_size: 128mb diff --git a/stacks/monitoring/glance.yaml b/stacks/monitoring/glance.yaml index 00ea1c3..a2db952 100644 --- a/stacks/monitoring/glance.yaml +++ b/stacks/monitoring/glance.yaml @@ -1,7 +1,7 @@ services: glance: container_name: glance - image: glanceapp/glance + image: glanceapp/glance:latest restart: unless-stopped volumes: - /srv/homelab/config/monitoring/glance:/app/config diff --git a/stacks/monitoring/grafana.yaml b/stacks/monitoring/grafana.yaml index 59a5ed4..c3499c9 100644 --- a/stacks/monitoring/grafana.yaml +++ b/stacks/monitoring/grafana.yaml @@ -2,7 +2,12 @@ # sudo chown -R 65534:65534 /srv/homelab/data/monitoring/prometheus # sudo chown -R 10001:10001 /srv/homelab/data/monitoring/loki # sudo chown -R 472:472 /srv/homelab/data/monitoring/grafana +# sudo chown -R 10001:10001 /srv/homelab/data/monitoring/alloy # +networks: + default: + name: monitoring + services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest @@ -78,7 +83,7 @@ services: max-file: "5" loki: - image: grafana/loki:2.9.0 + image: grafana/loki:3.6.0 container_name: loki ports: - "3100:3100" @@ -93,14 +98,17 @@ services: max-size: "10m" max-file: "5" - promtail: - image: grafana/promtail:2.9.0 - container_name: promtail - command: -config.file=/etc/promtail/config.yaml + alloy: + image: grafana/alloy:latest + container_name: alloy + command: run --storage.path=/var/lib/alloy /etc/alloy/config.alloy depends_on: - loki + ports: + - "12345:12345" volumes: - - /srv/homelab/config/monitoring/promtail-config.yaml:/etc/promtail/config.yaml:ro + - /srv/homelab/config/monitoring/alloy.alloy:/etc/alloy/config.alloy:ro + - /srv/homelab/data/monitoring/alloy:/var/lib/alloy - /var/log:/var/log:ro - /etc/machine-id:/etc/machine-id:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro diff --git a/stacks/monitoring/portainer.yaml b/stacks/monitoring/portainer.yaml index bf91f78..c7debcb 100644 --- a/stacks/monitoring/portainer.yaml +++ b/stacks/monitoring/portainer.yaml @@ -1,7 +1,7 @@ services: portainer: container_name: portainer - image: portainer/portainer-ce:lts + image: portainer/portainer-ce:latest restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/stacks/storage/gitea.yaml b/stacks/storage/gitea.yaml index fb34c55..94eeb6a 100644 --- a/stacks/storage/gitea.yaml +++ b/stacks/storage/gitea.yaml @@ -4,7 +4,7 @@ networks: services: server: - image: docker.gitea.com/gitea:1.25.4 + image: docker.gitea.com/gitea:latest container_name: gitea environment: - USER_UID=1000