init

alertmanager/alertmanager.yml

@@ -0,0 +1,17 @@
+global:
+  smtp_smarthost: 'smtp.gmail.com:587'
+  smtp_from: 'liuvanquyet@gkcorp.com.vn'
+  smtp_auth_username: 'liuvanquyet@gkcorp.com.vn'
+  smtp_auth_password: 'vejc mzti ming zvpj'  # Mã ứng dụng, KHÔNG phải mật khẩu Gmail thường
+
+route:
+  receiver: email-notification
+  group_wait: 10s
+  group_interval: 5m
+  repeat_interval: 1h
+
+receivers:
+  - name: email-notification
+    email_configs:
+      - to: 'quyetluu217@gmail.com'
+        send_resolved: true

docker-compose.yml

@@ -0,0 +1,102 @@
+version: '3.8'
+
+services:
+
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    restart: unless-stopped
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus/alert.rules.yml:/etc/prometheus/alert.rules.yml
+    networks:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=1qaz2wsxA@
+    volumes:
+      - grafana-data:/var/lib/grafana
+    networks:
+      - monitoring
+
+  node-exporter:
+    image: prom/node-exporter
+    container_name: node-exporter
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - '--path.rootfs=/rootfs'
+    networks:
+      - monitoring
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    restart: unless-stopped
+    ports:
+      - "8080:8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - monitoring
+
+  alertmanager:
+    image: prom/alertmanager
+    container_name: alertmanager
+    restart: unless-stopped
+    ports:
+      - "9093:9093"
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+    networks:
+      - monitoring
+
+  loki:
+    image: grafana/loki:main
+    container_name: loki
+    restart: unless-stopped
+    user: "1000:1000"
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml -target=all
+    volumes:
+      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml
+      - ./loki-data:/loki
+    networks:
+      - monitoring
+
+  promtail:
+    image: grafana/promtail:2.9.4
+    container_name: promtail
+    restart: unless-stopped
+    volumes:
+      - /var/log:/var/log
+      - /etc/machine-id:/etc/machine-id:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - ./promtail/promtail.yaml:/etc/promtail/promtail.yaml
+    networks:
+      - monitoring
+
+volumes:
+  grafana-data:
+
+networks:
+  monitoring:
+    driver: bridge

loki-data/chunks/loki_cluster_seed.json

@@ -0,0 +1 @@
+{"UID":"32e5c18c-5971-4fc9-b3b9-23fb018407a4","created_at":"2025-06-16T07:03:02.955281653Z","version":{"version":"2.9.4","revision":"f599ebc535","branch":"HEAD","buildUser":"root@5d9969758d88","buildDate":"2024-01-24T16:02:36Z","goVersion":"go1.21.3"}}

loki-data/index/uploader/name

@@ -0,0 +1 @@
+7fa8a79d2ac0-1750056575467132801

loki/loki-config.yaml

@@ -0,0 +1,61 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  wal:
+    enabled: true
+    dir: /loki/wal
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 3m
+  chunk_retain_period: 1m
+
+schema_config:
+  configs:
+    - from: 2023-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/boltdb-cache
+  filesystem:
+    directory: /loki/chunks
+
+compactor:
+  working_directory: /loki/compactor
+  compaction_interval: 1m
+  retention_enabled: true
+  delete_request_store: filesystem   # ✅ Bắt buộc nếu retention bật
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  volume_enabled: true
+  allow_structured_metadata: false   # ✅ Tắt yêu cầu schema v13 và tsdb
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://alertmanager:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_alertmanager_v2: true
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 168h

prometheus/alert.rules.yml

@@ -0,0 +1,14 @@
+groups:
+  - name: system-alerts
+    rules:
+      - alert: HighCPUUsage
+        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100) > 80
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "⚠️ CẢNH BÁO: CPU cao trên {{ $labels.instance }}"
+          description: |
+            💥 Máy chủ: {{ $labels.instance }}
+            🔥 CPU đã vượt ngưỡng 80% trong hơn 1 phút
+            📊 Giá trị hiện tại: {{ $value | printf "%.2f" }}%

prometheus/prometheus.yml

@@ -0,0 +1,33 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - 'alertmanager:9093'
+
+rule_files:
+  - "alert.rules.yml"
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['prometheus:9090']
+
+  - job_name: 'node-exporter'
+    static_configs:
+      - targets: ['node-exporter:9100']
+
+  - job_name: 'cadvisor'
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: 'grafana'
+    static_configs:
+      - targets: ['grafana:3000']
+
+  - job_name: 'loki'
+    static_configs:
+      - targets: ['loki:3100']

promtail/promtail.yaml

@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: varlogs
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: system
+          __path__: /var/log/*.log