┌──────────────────────────────────────────────────────────────┐
│                   MONITORING LAYERS                           │
└──────────────────────────────────────────────────────────────┘

┌──────────────────────────────────────────────────────────────┐
│ LAYER 1: INFRASTRUCTURE METRICS (Netdata)                    │
├──────────────────────────────────────────────────────────────┤
│ Production App Server (10.0.1.2):                            │
│  • CPU usage per core, load average                          │
│  • RAM usage, swap, cache                                    │
│  • Disk I/O, read/write rates                                │
│  • Network traffic, connections                              │
│  • Docker container metrics                                  │
│                                                              │
│ Database Server (10.0.1.3):                                  │
│  • PostgreSQL connections, queries/sec                       │
│  • Redis memory usage, hit rate                              │
│  • Disk space, backup storage                                │
│                                                              │
│ CI/CD Server + Staging Server: Same metrics                 │
│                                                              │
│ Real-time: 1-second intervals                                │
│ Retention: 14 days (on-disk)                                 │
└──────────────────────────────────────────────────────────────┘
                          ↓
┌──────────────────────────────────────────────────────────────┐
│ LAYER 2: APPLICATION HEALTH (UptimeRobot)                    │
├──────────────────────────────────────────────────────────────┤
│ HTTPS Monitors (5-minute checks):                            │
│  • https://app.your-domain.com/api/health                    │
│  • https://api.your-domain.com/api/health                    │
│  • https://www.your-domain.com/api/health                    │
│  • https://staging.your-domain.com/api/health                │
│                                                              │
│ Keyword Monitors (check for "ok" in response):              │
│  • Verify health endpoint returns valid JSON                 │
│                                                              │
│ Alerts: Email, Slack, SMS (paid)                             │
│ Status Page: Public uptime history                           │
└──────────────────────────────────────────────────────────────┘
                          ↓
┌──────────────────────────────────────────────────────────────┐
│ LAYER 3: ERROR TRACKING (Sentry)                             │
├──────────────────────────────────────────────────────────────┤
│ Application Errors:                                          │
│  • JavaScript errors (frontend)                              │
│  • API errors (backend)                                      │
│  • Database errors                                           │
│  • Authentication failures                                   │
│                                                              │
│ Performance Monitoring:                                      │
│  • Slow API endpoints (>1s)                                  │
│  • Database query performance                                │
│  • Page load times                                           │
│                                                              │
│ User Context: Session ID, user ID, browser info              │
│ Source Maps: Stack traces with original code                │
└──────────────────────────────────────────────────────────────┘
                          ↓
┌──────────────────────────────────────────────────────────────┐
│ LAYER 4: LOG AGGREGATION (Docker Logs)                       │
├──────────────────────────────────────────────────────────────┤
│ Container Logs (JSON format):                                │
│  • Application logs (stdout/stderr)                          │
│  • Access logs (Traefik)                                     │
│  • Database logs (PostgreSQL)                                │
│  • Cache logs (Redis)                                        │
│                                                              │
│ Log Rotation: 10MB max size, 5 files retained               │
│ Retention: 7 days local storage                              │
│                                                              │
│ Optional: Loki + Grafana for advanced querying              │
└──────────────────────────────────────────────────────────────┘
                          ↓
┌──────────────────────────────────────────────────────────────┐
│ ALERT ROUTING                                                │
├──────────────────────────────────────────────────────────────┤
│ Critical (Immediate): Slack + Email + SMS (optional)         │
│  • All services down                                         │
│  • Database unreachable                                      │
│  • Disk >95% full                                            │
│                                                              │
│ High (15 min): Slack + Email                                 │
│  • Single service down                                       │
│  • CPU >90% sustained                                        │
│  • High error rate (>5%)                                     │
│                                                              │
│ Medium (1 hour): Email                                       │
│  • High CPU (>80%)                                           │
│  • Slow API endpoints                                        │
│  • Elevated error rate (>1%)                                 │
│                                                              │
│ Low (24 hour digest): Email                                  │
│  • Deprecation warnings                                      │
│  • Info logs                                                 │
└──────────────────────────────────────────────────────────────┘

# SSH into server
ssh root@server-ip

# Install Netdata (auto-detects OS and dependencies)
bash <(curl -Ss https://my-netdata.io/kickstart.sh)

# Answer prompts:
# - Install required packages? Yes
# - Enable telemetry? No (privacy)
# - Claim to Netdata Cloud? Optional (for team dashboards)

# Verify installation
systemctl status netdata

# Access local dashboard
# http://server-ip:19999

# High CPU Usage Alert
alarm: high_cpu_usage
    on: system.cpu
  lookup: average -3m percentage foreach user,system
   every: 1m
    warn: $this > 80
    crit: $this > 95
    info: CPU usage is critically high ($this%)
      to: sysadmin

# Low Disk Space Alert
alarm: low_disk_space
    on: disk.space
  lookup: average -1m percentage of used
   every: 1m
    warn: $this > 80
    crit: $this > 90
    info: Disk space is running low ($this% used)
      to: sysadmin

# High RAM Usage Alert
alarm: high_memory_usage
    on: system.ram
  lookup: average -3m percentage
   every: 1m
    warn: $this > 85
    crit: $this > 95
    info: RAM usage is critically high ($this%)
      to: sysadmin

# High Load Average Alert
alarm: high_load_average
    on: system.load
  lookup: average -5m
   every: 1m
    warn: $this > 3
    crit: $this > 5
    info: System load average is high ($this)
      to: sysadmin

# PostgreSQL Connection Alert (DB server only)
alarm: high_postgres_connections
    on: postgres.connections_utilization
  lookup: average -1m
   every: 1m
    warn: $this > 80
    crit: $this > 95
    info: PostgreSQL connections are at $this% capacity
      to: dba

# Redis Memory Alert (DB server only)
alarm: high_redis_memory
    on: redis.memory
  lookup: average -1m
   every: 1m
    warn: $this > 768MB
    crit: $this > 900MB
    info: Redis memory usage is $this
      to: dba

# Docker Container Down Alert
alarm: docker_container_down
    on: docker.containers
  lookup: average -30s
   every: 10s
    crit: $this == 0
    info: Docker container stopped unexpectedly
      to: sysadmin

# Restart Netdata to apply alerts
systemctl restart netdata

# Verify alerts loaded
curl http://localhost:19999/api/v1/alarms

# Enable Slack notifications
SEND_SLACK="YES"

# Slack webhook URL (create in Slack: Apps → Incoming Webhooks)
SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

# Default Slack channel
DEFAULT_RECIPIENT_SLACK="#alerts"

# Role-specific channels
role_recipients_slack[sysadmin]="#ops-alerts"
role_recipients_slack[dba]="#database-alerts"

# Customize notification format
SLACK_MESSAGE_FORMAT='\
{ \
  "text": "${status} - ${alarm}", \
  "blocks": [ \
    { \
      "type": "section", \
      "text": { \
        "type": "mrkdwn", \
        "text": "*${status}* - ${alarm}" \
      } \
    }, \
    { \
      "type": "section", \
      "fields": [ \
        { \
          "type": "mrkdwn", \
          "text": "*Server:*\n${host}" \
        }, \
        { \
          "type": "mrkdwn", \
          "text": "*Value:*\n${value}" \
        }, \
        { \
          "type": "mrkdwn", \
          "text": "*Severity:*\n${severity}" \
        }, \
        { \
          "type": "mrkdwn", \
          "text": "*Time:*\n${date} ${time}" \
        } \
      ] \
    } \
  ] \
}'

# Send test notification
/usr/libexec/netdata/plugins.d/alarm-notify.sh test

# Check Slack channel for test message

# Claim each server to Netdata Cloud
netdata-claim.sh -token=YOUR_CLAIM_TOKEN -rooms=YOUR_ROOM_ID -url=https://app.netdata.cloud

# Access team dashboard
# https://app.netdata.cloud

Monitor Type: HTTPS
URL: https://app.your-domain.com/api/health
Friendly Name: Production App (Health)
Monitoring Interval: 5 minutes
Monitor Timeout: 30 seconds

HTTP Settings:
  Method: GET
  Expected Status: 200

Keyword Settings:
  Keyword Type: Keyword exists
  Keyword: "ok" # Verify JSON contains "ok" status

Alert Contacts:
  - Email: admin@your-domain.com
  - Slack: #alerts (via webhook)
  - SMS: +1234567890 (Pro plan only)

Monitor Type: HTTPS
URL: https://api.your-domain.com/api/health
Friendly Name: Production API (Health)
Monitoring Interval: 5 minutes
Expected Status: 200
Keyword: "ok"

Monitor Type: HTTPS
URL: https://www.your-domain.com/api/health
Friendly Name: Production Web (Health)
Monitoring Interval: 5 minutes
Expected Status: 200
Keyword: "ok"

Monitor Type: HTTPS
URL: https://staging.your-domain.com/api/health
Friendly Name: Staging (Health)
Monitoring Interval: 5 minutes
Expected Status: 200
Keyword: "ok"

Monitor Type: Port
IP/Host: 10.0.1.3
Port: 5432
Friendly Name: PostgreSQL (Port Check)
Monitoring Interval: 5 minutes

Monitor Type: Port
IP/Host: 10.0.1.3
Port: 6379
Friendly Name: Redis (Port Check)
Monitoring Interval: 5 minutes

Settings → Alert Contacts → Add Alert Contact

Contact Type: Web-Hook
Friendly Name: Slack #alerts
URL to Notify: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
POST Value (JSON):
{
  "text": "*monitorFriendlyName* is *alertTypeFriendlyName*",
  "blocks": [
    {
      "type": "section",
      "text": {
        "type": "mrkdwn",
        "text": "*alertTypeFriendlyName*: *monitorFriendlyName*"
      }
    },
    {
      "type": "section",
      "fields": [
        {
          "type": "mrkdwn",
          "text": "*URL:*\nmonitorURL"
        },
        {
          "type": "mrkdwn",
          "text": "*Time:*\nalertDateTime"
        },
        {
          "type": "mrkdwn",
          "text": "*Reason:*\nalertDetails"
        }
      ]
    }
  ]
}

Send when: Down, Up

UptimeRobot → Status Pages → Create Status Page

Status Page Name: RippleCore Status
Custom URL: ripplecore-status
Monitors to Display:
  - Production App
  - Production API
  - Marketing Website

Design:
  Logo: Upload your logo
  Colors: Purple (#0d1594), Teal (#26dbd9)
  Language: English

Public URL: https://stats.uptimerobot.com/ripplecore-status

<!-- apps/web/components/footer.tsx -->
<a href="https://stats.uptimerobot.com/ripplecore-status" target="_blank">
  System Status
</a>

import * as Sentry from "@sentry/nextjs";

Sentry.init({
  dsn: process.env.SENTRY_DSN,
  environment: process.env.NODE_ENV,

  // Performance monitoring
  tracesSampleRate: process.env.NODE_ENV === "production" ? 0.1 : 1.0,

  // Session replay (user interactions)
  replaysSessionSampleRate: 0.1,
  replaysOnErrorSampleRate: 1.0,

  // Error filtering
  beforeSend(event, hint) {
    // Filter out low-priority errors
    if (event.exception?.values?.[0]?.type === "ChunkLoadError") {
      return null; // Ignore chunk load errors (user navigated away)
    }
    return event;
  },
});

# apps/app/.env.local (and api, web)
SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id
SENTRY_ORG=your-org
SENTRY_PROJECT=ripplecore-app

Alert Name: High Error Rate (Production)
Environment: production
Conditions:
  - When: The issue is seen more than 100 times in 1 hour
  - And: Issue is unresolved

Actions:
  - Send notification to: #alerts (Slack)
  - Send email to: admin@your-domain.com

Alert Name: New Error Type (Production)
Environment: production
Conditions:
  - When: A new issue is created
  - And: Environment equals production

Actions:
  - Send notification to: #alerts (Slack)

Alert Name: Slow API Endpoints
Environment: production
Conditions:
  - When: The transaction duration is greater than 1000ms
  - And: Seen more than 50 times in 10 minutes

Actions:
  - Send notification to: #performance-alerts (Slack)

# In each server's /etc/netdata/health_alarm_notify.conf
SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

# Role-based routing
role_recipients_slack[sysadmin]="#ops-alerts"
role_recipients_slack[dba]="#database-alerts"

Critical Alerts:
  - admin@your-domain.com
  - oncall@your-domain.com

High Priority:
  - admin@your-domain.com
  - devops@your-domain.com

Medium/Low Priority:
  - devops@your-domain.com (daily digest)

Subject: [UP/DOWN] Production App is down

Monitor: Production App (Health)
URL: https://app.your-domain.com/api/health
Status: Down
Reason: Connection timeout after 30 seconds
Time: 2025-01-23 14:30:00 UTC

View details: https://uptimerobot.com/monitor/12345

# Install Grafana
docker run -d \
  --name grafana \
  -p 3000:3000 \
  -v grafana-data:/var/lib/grafana \
  grafana/grafana:latest

# Access: http://cicd-server-ip:3000
# Default login: admin/admin

services:
  app:
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "5"
        compress: "true"
        labels: "service,environment"

# View live logs for specific container
docker logs ripplecore-app --follow --tail 100

# Filter by severity
docker logs ripplecore-app --follow | grep ERROR

# Search for specific pattern
docker logs ripplecore-app --since 1h | grep "authentication failed"

# View all container logs
docker compose logs --follow

# Filter by service
docker compose logs app --follow

# docker-compose.yml (on CI/CD server)
version: '3.8'

services:
  loki:
    image: grafana/loki:2.9.0
    ports:
      - "3100:3100"
    volumes:
      - loki-data:/loki
    command: -config.file=/etc/loki/local-config.yaml

  promtail:
    image: grafana/promtail:2.9.0
    volumes:
      - /var/log:/var/log
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - ./promtail-config.yml:/etc/promtail/config.yml
    command: -config.file=/etc/promtail/config.yml

volumes:
  loki-data:

# All errors in last hour
{job="docker"} |= "ERROR" | json

# Slow API requests
{job="docker", container="ripplecore-app"}
  | json
  | duration > 1s

# Authentication failures
{job="docker"} |= "authentication failed"
  | json
  | line_format "{{.userId}} - {{.message}}"

#!/bin/bash
# Run daily at 9 AM via cron

# Check Netdata is running on all servers
servers=("10.0.1.2" "10.0.1.3" "10.0.1.4" "10.0.2.2")
for server in "${servers[@]}"; do
  if curl -f http://$server:19999/api/v1/info > /dev/null 2>&1; then
    echo "✅ Netdata running on $server"
  else
    echo "❌ Netdata down on $server" | mail -s "Alert: Netdata Down" admin@your-domain.com
  fi
done

# Check UptimeRobot monitors
uptime_api_key="YOUR_API_KEY"
curl -X POST https://api.uptimerobot.com/v2/getMonitors \
  -d "api_key=$uptime_api_key&format=json" \
  | jq '.monitors[] | select(.status != 2) | {name: .friendly_name, status: .status}'

# Check Sentry error rate
# (Use Sentry API to fetch error counts)

systemctl status netdata
# Output: Failed to start netdata

# Check logs
journalctl -u netdata -n 50

# Common issues:
# 1. Port 19999 already in use
sudo lsof -i :19999
sudo kill <PID>

# 2. Permissions issue
sudo chown -R netdata:netdata /var/lib/netdata
sudo chown -R netdata:netdata /var/cache/netdata

# Restart
sudo systemctl restart netdata

# Check alert configuration loaded
curl http://localhost:19999/api/v1/alarms | jq '.alarms[] | select(.name == "high_cpu_usage")'

# Verify Slack webhook
curl -X POST https://hooks.slack.com/services/YOUR/WEBHOOK/URL \
  -H 'Content-Type: application/json' \
  -d '{"text": "Test alert"}'

# Check notification config
cat /etc/netdata/health_alarm_notify.conf | grep SLACK

# Test health endpoint manually
time curl -v https://app.your-domain.com/api/health

# Verify response contains "ok"
curl -s https://app.your-domain.com/api/health | grep -o '"status":"ok"'

# Check SSL certificate
curl -vI https://app.your-domain.com 2>&1 | grep "SSL certificate"