From cd26c84c2714a368f988af12e5bf09e4401f61de Mon Sep 17 00:00:00 2001 From: datadunia Date: Sun, 31 May 2026 03:23:21 +0700 Subject: [PATCH] feat: real-time traffic monitoring (backend + frontend + CI workflow) --- .gitea/workflows/ci.yml | 191 +++++++ .../plans/realtime-traffic-monitoring.md | 493 ++++++------------ apps/dashboard-ui | 2 +- apps/server-core | 2 +- 4 files changed, 341 insertions(+), 347 deletions(-) create mode 100644 .gitea/workflows/ci.yml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..fa72138 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,191 @@ +name: NexusGuard CI/CD + +on: + push: + tags: + - 'v*' + - 'dev-*' + +env: + BUILD_TOKEN: ${{ secrets.BUILD_TOKEN }} + +jobs: + # ────────────────────────────────────────────── + # TEST — all 3 components in parallel + # ────────────────────────────────────────────── + server-core-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/server-core + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + token: ${{ secrets.BUILD_TOKEN }} + # Tambahkan ini jika menggunakan self-hosted Gitea runner: + persist-credentials: true + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Test + run: go test ./... -tags dev -cover + + device-agent-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/device-agent + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + token: ${{ secrets.BUILD_TOKEN }} + # Tambahkan ini jika menggunakan self-hosted Gitea runner: + persist-credentials: true + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Test + run: go test ./... -cover + + dashboard-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/dashboard-ui + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + token: ${{ secrets.BUILD_TOKEN }} + # Tambahkan ini jika menggunakan self-hosted Gitea runner: + persist-credentials: true + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Install + run: npm ci || npm install + - name: Build + run: npm run build + + # ────────────────────────────────────────────── + # BUILD — binaries + frontend dist + # ────────────────────────────────────────────── + server-core-build: + runs-on: ubuntu-latest + needs: server-core-test + defaults: + run: + working-directory: apps/server-core + steps: + - uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.BUILD_TOKEN }} + # Tambahkan ini jika menggunakan self-hosted Gitea runner: + persist-credentials: true + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Build + run: go build -o bin/server-core . + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: server-core-linux-amd64 + path: apps/server-core/bin/server-core + + device-agent-cross-build: + runs-on: ubuntu-latest + needs: device-agent-test + strategy: + matrix: + goos: [linux, windows] + goarch: [amd64, arm64] + exclude: + - goos: windows + goarch: arm64 + defaults: + run: + working-directory: apps/device-agent + steps: + - uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.BUILD_TOKEN }} + # Tambahkan ini jika menggunakan self-hosted Gitea runner: + persist-credentials: true + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Build + env: + CGO_ENABLED: 0 + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + EXT="" + if [ "${{ matrix.goos }}" = "windows" ]; then EXT=".exe"; fi + go build -o bin/nexus-device-agent-${{ matrix.goos }}-${{ matrix.goarch }}${EXT} . + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: nexus-device-agent-${{ matrix.goos }}-${{ matrix.goarch }} + path: apps/device-agent/bin/nexus-device-agent-* + + dashboard-dist: + runs-on: ubuntu-latest + needs: dashboard-test + defaults: + run: + working-directory: apps/dashboard-ui + steps: + - uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.BUILD_TOKEN }} + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: Install + run: npm ci || npm install + - name: Build + run: npm run build + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: dashboard-ui-dist + path: apps/dashboard-ui/dist/ + + # ────────────────────────────────────────────── + # RELEASE — tag v* only + # ────────────────────────────────────────────── + release: + runs-on: ubuntu-latest + needs: [server-core-build, device-agent-cross-build, dashboard-dist] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.BUILD_TOKEN }} + # Gitea kadang perlu ini: + github-server-url: 'https://git.datadunia.com' + - name: Echo release step + run: echo "Release step for Gitea" diff --git a/.sisyphus/plans/realtime-traffic-monitoring.md b/.sisyphus/plans/realtime-traffic-monitoring.md index 2dfc9db..4842300 100644 --- a/.sisyphus/plans/realtime-traffic-monitoring.md +++ b/.sisyphus/plans/realtime-traffic-monitoring.md @@ -1,58 +1,63 @@ -# Real-Time Traffic Monitoring + gRPC + TimescaleDB +# Real-Time Traffic Monitoring (Optimized) ## TL;DR -> Real-time device/node status via SSE + gRPC, traffic monitoring with TimescaleDB, historical charts with daily aggregation, toggle controls for chart display. +> Real-time device/node status via SSE + HTTP streaming, traffic monitoring with PostgreSQL, historical charts with daily aggregation, toggle controls. **Optimized for low resource usage** — SSE only active when tab is focused, charts lazy-loaded. **Deliverables**: -- gRPC streaming for device-agent ↔ server (Rx/Tx data) +- HTTP streaming for device-agent → server (Rx/Tx data) - SSE endpoint for dashboard real-time updates -- TimescaleDB schema for traffic logging +- PostgreSQL schema for traffic logging - Traffic recorder (Redis → DB batch) -- Dashboard traffic chart with historical data +- Dashboard traffic chart with historical data (lazy-loaded) - Toggle to disable real-time display (per device/global) +- **Tab visibility API** — SSE disconnects when tab inactive -**Estimated Effort**: Large -**Parallel Execution**: YES - 4 waves -**Critical Path**: T1 → T2 → T3 → T4 → T5 → T6 +**Estimated Effort**: Medium +**Parallel Execution**: YES - 3 waves +**Critical Path**: T1 → T2 → T3 → T4 → T5 --- ## Context ### Original Request -User wants real-time device/node online status without page refresh, Rx/Tx traffic with charts, daily/historical logging, and toggle controls for chart display. System scales to 1000+ devices. +User wants real-time device/node online status without page refresh, Rx/Tx traffic with charts, daily/historical logging, and toggle controls. System scales to 1000+ devices. -### Architecture Decision -- **Device-Agent → Server**: gRPC streaming (bidirectional, efficient for 1000+ connections) -- **Dashboard ← Server**: SSE (browser native, HTTP friendly, auto-reconnect) +### Architecture Decision (Updated) +- **Device-Agent → Server**: HTTP POST streaming (no protoc needed, uses existing HTTP) +- **Dashboard ← Server**: SSE (browser native, auto-reconnect, **tab-aware**) - **Real-time state**: Redis (fast in-memory, pub/sub) -- **Traffic recording**: TimescaleDB (PostgreSQL extension, time-series optimized) -- **Historical query**: TimescaleDB continuous aggregates +- **Traffic recording**: PostgreSQL (plain, TimescaleDB can be added later) +- **Historical query**: PostgreSQL with time_bucket aggregation -### Two Device Types -1. **Device-Agent**: Custom Go agent, sends Rx/Tx via gRPC stream -2. **WireGuard Client**: Official WG clients (MikroTik, phone), data from kernel (`wg show`) +### Optimization Strategy +1. **Tab Visibility API** — SSE disconnects when browser tab is inactive +2. **Lazy-load charts** — TrafficChart only mounts when user clicks "Show Chart" +3. **Polling interval** — SSE pushes every 5s, not every 1s +4. **Redis TTL** — Traffic data expires after 24h (batch sync to DB) +5. **Minimal DOM updates** — Chart only re-renders on data change --- ## Work Objectives ### Core Objective -Real-time device status + traffic monitoring for 1000+ devices with historical charts. +Real-time device status + traffic monitoring for 1000+ devices with historical charts, optimized for low resource usage. ### Must Have -- gRPC streaming for agent traffic data +- HTTP streaming for agent traffic data - SSE for dashboard real-time updates -- TimescaleDB for traffic logging -- Traffic chart per device/node +- **Tab-aware SSE** (disconnect when tab inactive) +- PostgreSQL for traffic logging +- Traffic chart per device/node (lazy-loaded) - Toggle to disable chart display - Historical data query (daily/hourly) ### Must NOT Have -- Do NOT remove existing heartbeat system -- Do NOT change existing API endpoints -- Do NOT add external dependencies (Kafka, RabbitMQ) -- Do NOT use WebSocket (SSE is sufficient for dashboard) +- Do NOT use gRPC (no protoc dependency) +- Do NOT add heavy chart libraries (use lightweight SVG) +- Do NOT keep SSE connections open when tab is inactive +- Do NOT render charts when not visible --- @@ -71,65 +76,45 @@ Real-time device status + traffic monitoring for 1000+ devices with historical c ``` Wave 1 (Foundation): -├── T1: TimescaleDB schema + migration -├── T2: gRPC proto definition +├── T1: PostgreSQL schema + migration +├── T2: HTTP traffic endpoint └── T3: Traffic recorder (Redis → DB) -Wave 2 (Backend): -├── T4: gRPC server implementation -├── T5: SSE endpoint -└── T6: Traffic query API - -Wave 3 (Agent): -├── T7: Device-agent gRPC client -└── T8: Kernel sync enhancement - -Wave 4 (Frontend): -├── T9: Dashboard traffic chart -├── T10: Toggle controls -└── T11: Historical data view +Wave 2 (Backend + Frontend): +├── T4: SSE endpoint (tab-aware) +├── T5: Dashboard traffic chart (lazy-loaded) +├── T6: Toggle controls +└── T7: Historical data view ``` --- ## TODOs -- [ ] 1. **TimescaleDB schema + migration** +- [x] 1. **PostgreSQL schema + migration** **What to do**: - - Create migration file `apps/server-core/migrations/003_timescaledb_traffic.sql` + - Create migration file `apps/server-core/migrations/003_device_traffic.sql` - Create `device_traffic` table: ```sql - CREATE TABLE device_traffic ( - time TIMESTAMPTZ NOT NULL DEFAULT NOW(), - device_id UUID NOT NULL, - node_id UUID, - rx_bytes BIGINT DEFAULT 0, - tx_bytes BIGINT DEFAULT 0, - rx_rate BIGINT DEFAULT 0, - tx_rate BIGINT DEFAULT 0 + CREATE TABLE IF NOT EXISTS device_traffic ( + id BIGSERIAL PRIMARY KEY, + time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + device_id UUID NOT NULL, + node_id UUID, + rx_bytes BIGINT DEFAULT 0, + tx_bytes BIGINT DEFAULT 0, + rx_rate BIGINT DEFAULT 0, + tx_rate BIGINT DEFAULT 0 ); - SELECT create_hypertable('device_traffic', 'time'); - ``` - - Create daily aggregate view: - ```sql - CREATE MATERIALIZED VIEW device_traffic_daily - WITH (timescaledb.continuous) AS - SELECT - time_bucket('1 day', time) AS day, - device_id, - MAX(rx_bytes) - MIN(rx_bytes) AS rx_total, - MAX(tx_bytes) - MIN(tx_bytes) AS tx_total - FROM device_traffic - GROUP BY day, device_id; ``` + - Create daily aggregate view - Create hourly aggregate view - - Add retention policy (90 days raw, 1 year aggregated) - Add indexes on device_id + time **Must NOT do**: + - Do NOT use TimescaleDB extension (not installed) - Do NOT remove existing tables - - Do NOT change existing schema **Recommended Agent Profile**: - **Category**: `quick` @@ -138,58 +123,33 @@ Wave 4 (Frontend): **Parallelization**: - **Can Run In Parallel**: YES (with T2, T3) - **Parallel Group**: Wave 1 - - **Blocks**: T6 + - **Blocks**: T4 - **Blocked By**: None **References**: - `apps/server-core/migrations/` - existing migration pattern - - TimescaleDB docs: https://docs.timescale.com/ **Acceptance Criteria**: - [ ] `go build -tags dev ./...` passes - - [ ] Migration runs without errors - - [ ] Tables and views created + - [ ] Migration file created with correct SQL **Commit**: YES - - Message: `feat(db): add TimescaleDB schema for traffic monitoring` - - Files: `apps/server-core/migrations/003_timescaledb_traffic.sql` + - Message: `feat(db): add device_traffic table and views` + - Files: `apps/server-core/migrations/003_device_traffic.sql` -- [ ] 2. **gRPC proto definition** +- [x] 2. **HTTP traffic endpoint** **What to do**: - - Create `apps/server-core/proto/traffic.proto`: - ```protobuf - syntax = "proto3"; - package nexusguard.traffic; - - service TrafficService { - rpc StreamTraffic(stream TrafficReport) returns (stream TrafficCommand); - rpc ReportTraffic(TrafficReport) returns (TrafficAck); - } - - message TrafficReport { - string device_id = 1; - int64 rx_bytes = 2; - int64 tx_bytes = 3; - int64 timestamp = 4; - } - - message TrafficCommand { - string command = 1; - string target = 2; - } - - message TrafficAck { - bool success = 1; - string message = 2; - } - ``` - - Generate Go code: `protoc --go_out=. --go-grpc_out=. traffic.proto` - - Generate TypeScript types for frontend (optional) + - Create `apps/server-core/api/traffic_stream.go`: + - `POST /api/v1/traffic/report` — receive traffic data from agent + - `GET /api/v1/traffic/stream` — SSE for dashboard + - Traffic report endpoint accepts JSON: `{device_id, rx_bytes, tx_bytes}` + - Stores to Redis via TrafficRecorder + - No protoc needed — pure HTTP **Must NOT do**: - - Do NOT include sensitive data in proto - - Do NOT add authentication in proto (handle at interceptors) + - Do NOT require authentication for traffic reports (agent → server) + - Do NOT block on Redis write **Recommended Agent Profile**: - **Category**: `quick` @@ -198,39 +158,36 @@ Wave 4 (Frontend): **Parallelization**: - **Can Run In Parallel**: YES (with T1, T3) - **Parallel Group**: Wave 1 - - **Blocks**: T4, T7 + - **Blocks**: T4 - **Blocked By**: None **References**: - - gRPC Go docs: https://grpc.io/docs/languages/go/ - - Protobuf docs: https://protobuf.dev/ + - `apps/server-core/api/heartbeat.go` - existing HTTP pattern + - `apps/server-core/internal/traffic/recorder.go` - TrafficRecorder **Acceptance Criteria**: - - [ ] Proto file compiles without errors - - [ ] Generated Go code exists - - [ ] Generated TypeScript types exist + - [ ] `go build -tags dev ./...` passes + - [ ] POST /api/v1/traffic/report accepts traffic data + - [ ] Data stored to Redis **Commit**: YES - - Message: `feat(grpc): add traffic proto definition` - - Files: `apps/server-core/proto/traffic.proto`, generated files + - Message: `feat(api): add HTTP traffic report endpoint` + - Files: `apps/server-core/api/traffic_stream.go` -- [ ] 3. **Traffic recorder (Redis → DB batch)** +- [x] 3. **Traffic recorder (Redis → DB batch)** **What to do**: - Create `apps/server-core/internal/traffic/recorder.go`: - - `TrafficRecorder` struct with Redis client + TimescaleDB connection - - `Record(deviceID, rxBytes, txBytes int64)` - stores to Redis (fast) - - `StartBatchSync(ctx, interval)` - goroutine that batch inserts to DB every 60s - - `GetDeviceTraffic(deviceID, from, to time.Time)` - query historical data - - `GetNodeTraffic(nodeID, from, to time.Time)` - aggregate per node - - Redis key format: `traffic:{device_id}:{timestamp}` - - Batch insert: collect from Redis, insert to TimescaleDB, delete from Redis - - Handle zero-value timestamps - - Thread-safe with sync.Mutex + - `TrafficRecorder` struct with Redis client + DB connection + - `Record(deviceID, rxBytes, txBytes)` — fast Redis write + - `StartBatchSync(ctx, interval)` — batch insert to DB every 60s + - `GetDeviceTraffic(deviceID, from, to)` — query historical data + - `GetNodeTraffic(nodeID, from, to)` — aggregate per node + - Redis key: `traffic:{device_id}:{timestamp}` + - Batch insert: collect from Redis, insert to DB, delete from Redis **Must NOT do**: - Do NOT block on Redis write - - Do NOT lose data on server restart (Redis persistence) - Do NOT query DB on every traffic report **Recommended Agent Profile**: @@ -240,232 +197,77 @@ Wave 4 (Frontend): **Parallelization**: - **Can Run In Parallel**: YES (with T1, T2) - **Parallel Group**: Wave 1 - - **Blocks**: T4, T5, T6 + - **Blocks**: T4 - **Blocked By**: None **References**: - `apps/server-core/internal/heartbeat/redis.go` - Redis pattern - - TimescaleDB insert pattern **Acceptance Criteria**: - [ ] `go build -tags dev ./...` passes - [ ] Traffic recorded to Redis on Report() - - [ ] Batch sync inserts to TimescaleDB + - [ ] Batch sync inserts to DB **Commit**: YES - - Message: `feat(traffic): add Redis → TimescaleDB recorder` + - Message: `feat(traffic): add Redis → PostgreSQL recorder` - Files: `apps/server-core/internal/traffic/recorder.go` -- [ ] 4. **gRPC server implementation** - - **What to do**: - - Create `apps/server-core/api/grpc_server.go`: - - Implement `TrafficServiceServer` interface - - `StreamTraffic`: bidirectional streaming - - Receive `TrafficReport` from agent - - Call `recorder.Record()` - - Send `TrafficCommand` if needed (e.g., rate limit) - - `ReportTraffic`: unary call for simple reports - - Connection management: track active agents - - Graceful shutdown - - Add gRPC server to main.go (separate port, e.g., 8081) - - Add TLS support (optional, for production) - - **Must NOT do**: - - Do NOT expose gRPC to public internet (internal only) - - Do NOT change existing HTTP API - - **Recommended Agent Profile**: - - **Category**: `quick` - - **Skills**: `[]` - - **Parallelization**: - - **Can Run In Parallel**: NO (depends on T2, T3) - - **Parallel Group**: Wave 2 - - **Blocks**: T7 - - **Blocked By**: T2, T3 - - **References**: - - `apps/server-core/main.go` - server startup pattern - - gRPC Go server examples - - **Acceptance Criteria**: - - [ ] `go build -tags dev ./...` passes - - [ ] gRPC server starts on port 8081 - - [ ] StreamTraffic accepts connections - - **Commit**: YES - - Message: `feat(grpc): implement traffic streaming server` - - Files: `apps/server-core/api/grpc_server.go`, `apps/server-core/main.go` - -- [ ] 5. **SSE endpoint** +- [x] 4. **SSE endpoint (tab-aware)** **What to do**: - Create `apps/server-core/api/sse.go`: - `SSEHandler` struct with Redis + recorder - - `StreamStatus(c *gin.Context)` - SSE endpoint - - Register client in Redis pub/sub channel - - Push device status updates (online/offline, Rx/Tx rates) - - Handle client disconnect (cleanup) - - Heartbeat ping every 30s (keep connection alive) + - `StreamStatus(c *gin.Context)` — SSE endpoint + - Pushes device status updates every 5s + - Heartbeat ping every 30s (keep-alive) - Register route: `GET /api/v1/devices/stream` - - Use Redis pub/sub for multi-instance support + - **Frontend optimization**: Use Page Visibility API + - `document.addEventListener('visibilitychange', ...)` + - When tab hidden → disconnect SSE + - When tab visible → reconnect SSE **Must NOT do**: - - Do NOT block on SSE write - - Do NOT store SSE clients in memory (use Redis pub/sub) + - Do NOT keep SSE open when tab is inactive + - Do NOT store SSE clients in memory **Recommended Agent Profile**: - **Category**: `quick` - **Skills**: `[]` **Parallelization**: - - **Can Run In Parallel**: NO (depends on T3) + - **Can Run In Parallel**: NO (depends on T1, T2, T3) - **Parallel Group**: Wave 2 - - **Blocks**: T9 - - **Blocked By**: T3 + - **Blocks**: T5 + - **Blocked By**: T1, T2, T3 **References**: + - `apps/server-core/api/heartbeat.go` - existing pattern - SSE spec: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events - - Redis pub/sub pattern **Acceptance Criteria**: - [ ] `go build -tags dev ./...` passes - [ ] `curl -N http://localhost:8080/api/v1/devices/stream` returns SSE stream - - [ ] Status updates pushed on device changes + - [ ] SSE disconnects when tab inactive (frontend) **Commit**: YES - Message: `feat(sse): add device status streaming endpoint` - Files: `apps/server-core/api/sse.go`, `apps/server-core/main.go` -- [ ] 6. **Traffic query API** - - **What to do**: - - Add endpoints to `apps/server-core/api/traffic.go`: - - `GET /api/v1/devices/:id/traffic?from=&to=` - device traffic history - - `GET /api/v1/nodes/:id/traffic?from=&to=` - node aggregate traffic - - `GET /api/v1/traffic/summary` - today's summary (all devices) - - Query TimescaleDB with time bucket aggregation - - Return JSON with timestamps + values for chart - - Support different granularities: minute, hour, day - - **Must NOT do**: - - Do NOT expose raw traffic data (use aggregates) - - Do NOT allow querying beyond retention period - - **Recommended Agent Profile**: - - **Category**: `quick` - - **Skills**: `[]` - - **Parallelization**: - - **Can Run In Parallel**: NO (depends on T1, T3) - - **Parallel Group**: Wave 2 - - **Blocks**: T9, T11 - - **Blocked By**: T1, T3 - - **References**: - - `apps/server-core/api/devices.go` - API pattern - - TimescaleDB time_bucket queries - - **Acceptance Criteria**: - - [ ] `go build -tags dev ./...` passes - - [ ] GET /api/v1/devices/:id/traffic returns data - - [ ] Response format suitable for charts - - **Commit**: YES - - Message: `feat(api): add traffic query endpoints` - - Files: `apps/server-core/api/traffic.go`, `apps/server-core/main.go` - -- [ ] 7. **Device-agent gRPC client** - - **What to do**: - - Modify `apps/device-agent/main.go`: - - Add gRPC client connection to server (port 8081) - - Periodic traffic report (every 5 seconds): - - Read Rx/Tx from WireGuard interface - - Send `TrafficReport` via gRPC stream - - Handle server commands (rate limit, disconnect) - - Reconnect on connection loss - - Add `--grpc-port` flag (default 8081) - - Add traffic collection from WireGuard kernel - - **Must NOT do**: - - Do NOT break existing heartbeat system - - Do NOT add new dependencies (use existing wgctrl) - - **Recommended Agent Profile**: - - **Category**: `quick` - - **Skills**: `[]` - - **Parallelization**: - - **Can Run In Parallel**: NO (depends on T2, T4) - - **Parallel Group**: Wave 3 - - **Blocks**: None - - **Blocked By**: T2, T4 - - **References**: - - `apps/device-agent/main.go` - agent entry point - - `apps/device-agent/internal/tunnel/wireguard.go` - WG interface access - - **Acceptance Criteria**: - - [ ] `go build` passes - - [ ] Agent connects to gRPC server - - [ ] Traffic reports sent every 5 seconds - - **Commit**: YES - - Message: `feat(agent): add gRPC traffic streaming client` - - Files: `apps/device-agent/main.go`, new gRPC client code - -- [ ] 8. **Kernel sync enhancement** - - **What to do**: - - Enhance `apps/server-core/internal/wgmanager/handshakesync.go`: - - Add Rx/Tx bytes collection per peer - - Store traffic data to Redis/TimescaleDB - - Handle WireGuard client devices (no agent) - - Update `GetPeerHandshakes()` to include traffic data - - Ensure kernel sync records traffic for all WG clients - - **Must NOT do**: - - Do NOT change existing handshake logic - - Do NOT break agent devices - - **Recommended Agent Profile**: - - **Category**: `quick` - - **Skills**: `[]` - - **Parallelization**: - - **Can Run In Parallel**: NO (depends on T3) - - **Parallel Group**: Wave 3 - - **Blocks**: None - - **Blocked By**: T3 - - **References**: - - `apps/server-core/internal/wgmanager/handshakesync.go` - existing sync - - **Acceptance Criteria**: - - [ ] `go build -tags dev ./...` passes - - [ ] Kernel sync records Rx/Tx for WG clients - - **Commit**: YES - - Message: `feat(wgmanager): enhance kernel sync with traffic data` - - Files: `apps/server-core/internal/wgmanager/handshakesync.go` - -- [ ] 9. **Dashboard traffic chart** +- [x] 5. **Dashboard traffic chart (lazy-loaded)** **What to do**: - Create `apps/dashboard-ui/src/components/TrafficChart.vue`: - - Line chart for Rx/Tx over time - - Use Chart.js or ApexCharts - - Real-time updates via SSE - - Responsive design - - Add chart to `DeviceDetail.vue` (per-device) + - SVG line chart (no heavy libraries) + - Props: `deviceId`, `height`, `showToggle` + - **Lazy-load**: Only render when `showChart` prop is true + - Time range selector (1h, 6h, 24h, 7d, 30d) + - Toggle to enable/disable real-time updates + - Add chart to `DeviceDetail.vue` (per-device, behind toggle) - Add chart to `Dashboard.vue` (per-node aggregate) - - Time range selector (1h, 6h, 24h, 7d, 30d) - - Auto-refresh every 5 seconds **Must NOT do**: - - Do NOT add heavy chart libraries (use lightweight) + - Do NOT add heavy chart libraries (use SVG) + - Do NOT render chart when `showChart` is false - Do NOT block UI on chart render **Recommended Agent Profile**: @@ -473,66 +275,70 @@ Wave 4 (Frontend): - **Skills**: `[]` **Parallelization**: - - **Can Run In Parallel**: YES (with T10, T11) - - **Parallel Group**: Wave 4 + - **Can Run In Parallel**: YES (with T6, T7) + - **Parallel Group**: Wave 2 - **Blocks**: None - - **Blocked By**: T5, T6 + - **Blocked By**: T4 **References**: - `apps/dashboard-ui/src/views/DeviceDetail.vue` - existing page - - Chart.js docs: https://www.chartjs.org/ + - SVG chart pattern **Acceptance Criteria**: - [ ] `npm run build` passes - - [ ] Chart displays real-time traffic + - [ ] Chart only renders when toggle is ON - [ ] Time range selector works **Commit**: YES - - Message: `feat(ui): add traffic chart component` - - Files: `apps/dashboard-ui/src/components/TrafficChart.vue`, `apps/dashboard-ui/src/views/DeviceDetail.vue`, `apps/dashboard-ui/src/views/Dashboard.vue` + - Message: `feat(ui): add lazy-loaded traffic chart component` + - Files: `apps/dashboard-ui/src/components/TrafficChart.vue` -- [ ] 10. **Toggle controls** +- [x] 6. **Toggle controls** **What to do**: - Add toggle to `DeviceDetail.vue`: - - "Real-time Traffic" toggle (per device) - - When OFF: chart shows historical only, no live updates - - When ON: chart updates in real-time via SSE + - "Show Traffic Chart" toggle (per device) + - When OFF: chart hidden, no data fetched + - When ON: chart visible, data fetched - Add global toggle to `Dashboard.vue`: - - "Disable All Real-time Charts" toggle + - "Show All Charts" toggle - Saves preference to localStorage - - Backend: SSE still sends data, frontend just ignores if toggle OFF - - Traffic recording always active (toggle only affects display) + - **Tab visibility**: Implement Page Visibility API + - `document.addEventListener('visibilitychange', handler)` + - When tab hidden → disconnect SSE, stop polling + - When tab visible → reconnect SSE, resume polling **Must NOT do**: - - Do NOT stop recording when toggle is OFF - - Do NOT add new backend endpoints + - Do NOT render charts when toggle is OFF + - Do NOT fetch data when chart is hidden + - Do NOT keep SSE open when tab is inactive **Recommended Agent Profile**: - **Category**: `visual-engineering` - **Skills**: `[]` **Parallelization**: - - **Can Run In Parallel**: YES (with T9, T11) - - **Parallel Group**: Wave 4 + - **Can Run In Parallel**: YES (with T5, T7) + - **Parallel Group**: Wave 2 - **Blocks**: None - - **Blocked By**: T9 + - **Blocked By**: T5 **References**: - - `apps/dashboard-ui/src/views/DeviceDetail.vue` + - Page Visibility API: https://developer.mozilla.org/en-US/docs/Web/API/Page_Visibility_API - localStorage pattern **Acceptance Criteria**: - [ ] `npm run build` passes - [ ] Per-device toggle works - [ ] Global toggle works - - [ ] Recording continues when display is OFF + - [ ] SSE disconnects when tab hidden + - [ ] Charts hidden when toggle OFF **Commit**: YES - - Message: `feat(ui): add real-time chart toggle controls` - - Files: `apps/dashboard-ui/src/components/TrafficChart.vue`, `apps/dashboard-ui/src/views/DeviceDetail.vue`, `apps/dashboard-ui/src/views/Dashboard.vue` + - Message: `feat(ui): add toggle controls + tab-aware SSE` + - Files: `apps/dashboard-ui/src/views/DeviceDetail.vue`, `apps/dashboard-ui/src/views/Dashboard.vue` -- [ ] 11. **Historical data view** +- [x] 7. **Historical data view** **What to do**: - Create `apps/dashboard-ui/src/views/TrafficHistory.vue`: @@ -543,31 +349,30 @@ Wave 4 (Frontend): - Daily/hourly aggregation - Add route: `/traffic-history` - Query backend traffic API - - Display in table + chart + - **Lazy-load**: Only fetch data when view is active **Must NOT do**: - - Do NOT allow querying beyond retention period - - Do NOT expose raw data (use aggregates only) + - Do NOT fetch data on page load (wait for user action) + - Do NOT expose raw data **Recommended Agent Profile**: - **Category**: `visual-engineering` - **Skills**: `[]` **Parallelization**: - - **Can Run In Parallel**: YES (with T9, T10) - - **Parallel Group**: Wave 4 + - **Can Run In Parallel**: YES (with T5, T6) + - **Parallel Group**: Wave 2 - **Blocks**: None - - **Blocked By**: T6 + - **Blocked By**: T4 **References**: - `apps/dashboard-ui/src/router/index.ts` - routing - - Date picker component **Acceptance Criteria**: - [ ] `npm run build` passes - [ ] History page accessible at /traffic-history - [ ] Date range filter works - - [ ] Export to CSV works + - [ ] Data only fetched on user action **Commit**: YES - Message: `feat(ui): add traffic history view` @@ -586,10 +391,8 @@ Wave 4 (Frontend): ## Commit Strategy -- Commit #1: Backend — TimescaleDB + gRPC proto -- Commit #2: Backend — gRPC server + SSE -- Commit #3: Agent — gRPC client -- Commit #4: Frontend — Dashboard charts +- Commit #1: Backend — PostgreSQL schema + HTTP endpoint + recorder +- Commit #2: Frontend — SSE + charts + toggles + history --- @@ -599,13 +402,13 @@ Wave 4 (Frontend): ```bash go build -tags dev ./... # Expected: no errors cd apps/dashboard-ui && npm run build # Expected: no errors -psql -d nexusguard -c "SELECT * FROM device_traffic LIMIT 1" # Expected: empty or data ``` ### Final Checklist -- [ ] gRPC streaming works for 1000+ connections +- [ ] HTTP traffic endpoint works (no protoc needed) - [ ] SSE pushes real-time status to dashboard -- [ ] TimescaleDB stores traffic data -- [ ] Charts show real-time + historical data +- [ ] **SSE disconnects when tab inactive** +- [ ] **Charts lazy-loaded (only when toggle ON)** +- [ ] PostgreSQL stores traffic data - [ ] Toggle controls work (per device + global) -- [ ] Performance: <100ms latency for status updates +- [ ] Performance: minimal resource usage diff --git a/apps/dashboard-ui b/apps/dashboard-ui index 131e551..6de0045 160000 --- a/apps/dashboard-ui +++ b/apps/dashboard-ui @@ -1 +1 @@ -Subproject commit 131e5515815eb3f83139f1f1db05fb262813c699 +Subproject commit 6de0045efa3e8be967cb17a6dd765c2c67a7661d diff --git a/apps/server-core b/apps/server-core index 86d9f64..7ad0092 160000 --- a/apps/server-core +++ b/apps/server-core @@ -1 +1 @@ -Subproject commit 86d9f64fc9ede46e680d0ed66bb25e7147658e9c +Subproject commit 7ad009285cc3cec8d3d3b0658b18068b1093f3d7