clickstack: C-full migratsioon — app-only + eraldi mongo + ametlik ClickStack OTel-collector (custom.config); eemalda all-in-one + vanad failid

dev/clickstack-entry.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-# ClickStack all-in-one käivitusskript: esmalt MongoDB, siis rakendus
-set -e
-cd /app
-mongod --dbpath /data/db --bind_ip 127.0.0.1 --fork --logpath /var/log/mongod.log
-# OTel kollektor (opampsupervisor): võtab API-lt konfiguratsiooni ja haldab otel_* tabeleid ClickHouse'is.
-# Ilma selleta ei püsi OTLP meetrikad/logid/trace'id (ainult API võtab vastu ja viskab ära).
-/otel-entrypoint.sh /usr/local/bin/opampsupervisor > /var/log/otel-collector.log 2>&1 &
-exec node ./node_modules/.bin/concurrently --kill-others-on-fail --names=API,APP,ALERT-TASK \
-  "PORT=8000 HYPERDX_APP_PORT=8080 ./packages/api/bin/hyperdx api" \
-  "cd ./packages/app/packages/app && HOSTNAME='0.0.0.0' HYPERDX_API_PORT=8000 PORT=8080 node server.js" \
-  "./packages/api/bin/hyperdx task check-alerts"

dev/config/otel-collector/config.yaml → dev/config/otel-collector/custom.config.yaml

@@ -1,3 +1,8 @@
+# ClickStack ametliku OTel-collector'i CUSTOM config (CUSTOM_OTELCOL_CONFIG_FILE).
+# Base config (OpAMP kaudu app'ist) defineerib juba: clickhouse eksportija, memory_limiter, batch,
+# OTLP receiver + põhipipeline'id. Siin lisame AINULT hosti-meetrikad + konteinerilogid.
+# NB: ei tohi baaskomponente üle kirjutada; pipeline'id viitavad base `clickhouse` eksportijale.
+
 receivers:
   prometheus:
     config:
@@ -14,9 +19,7 @@ receivers:
           static_configs:
             - targets: ["host.docker.internal:8070"]
 
-  # Konteinerite logid: Docker JSON-logid otse /var/lib/docker/containers/*.
-  # `container` parser võtab log keha + lisab container.name/id ja log.iostream.
-  # start_at: end -> ära loe ajalugu (ainult uued read alates käivitamisest).
+  # Konteinerite Docker JSON-logid.
   filelog:
     include:
       - /var/lib/docker/containers/*/*-json.log
@@ -26,23 +29,10 @@ receivers:
       - type: container
         id: container-parser
         format: docker
-        # Docker-teed ei vasta k8s-tee mustrile -> ilma selleta "failed to detect a valid log path".
         add_metadata_from_filepath: false
 
-  # Rakenduste trace'id (OTel auto-instrumentation) -> HyperDX Service Map.
-  otlp:
-    protocols:
-      grpc:
-        endpoint: 0.0.0.0:4317
-      http:
-        endpoint: 0.0.0.0:4318
-
 processors:
-  memory_limiter:
-    check_interval: 5s
-    limit_mib: 400
-    spike_limit_mib: 100
-  # Hoia ainult vajalikud metrikad (cadvisor vaikimisi maht > 4 MB -> HyperDX lükkab tagasi)
+  # Hoia vaid vajalikud metrikad (cadvisor vaikimisi maht > 4 MB).
   filter/keep:
     metrics:
       include:
@@ -51,10 +41,6 @@ processors:
           - "^node_(memory_MemAvailable_bytes|memory_MemTotal_bytes|memory_SwapFree_bytes|memory_SwapTotal_bytes|load1|cpu_seconds_total|filesystem_avail_bytes|filesystem_size_bytes|network_receive_bytes_total|network_transmit_bytes_total)$"
           - "^container_(cpu_usage_seconds_total|memory_working_set_bytes|memory_usage_bytes|network_receive_bytes_total|network_transmit_bytes_total)$"
           - ".*llamacpp.*"
-  batch:
-    send_batch_size: 500
-    timeout: 10s
-  # Filtreeri välja tuntud, kahjutud mürad.
   # cAdvisor logib btrfs hoiatuse (konteiner ei näe /dev/mapper/data-root) — benignne.
   filter/logs:
     error_mode: ignore
@@ -62,26 +48,13 @@ processors:
       log_record:
         - 'IsMatch(body, "Unable to get btrfs mountpoint IDs")'
 
-exporters:
-  otlp:
-    endpoint: clickstack:4317
-    tls:
-      insecure: true
-    headers:
-      authorization: ${env:HYPERDX_INGEST_KEY}
-    compression: none
-
 service:
   pipelines:
-    metrics:
+    metrics/ohmu:
       receivers: [prometheus]
       processors: [memory_limiter, filter/keep, batch]
-      exporters: [otlp]
-    logs:
+      exporters: [clickhouse]
+    logs/ohmu:
       receivers: [filelog]
       processors: [memory_limiter, filter/logs, batch]
-      exporters: [otlp]
-    traces:
-      receivers: [otlp]
-      processors: [memory_limiter, batch]
-      exporters: [otlp]
+      exporters: [clickhouse]

dev/config/otel-ratelimit/nginx.conf

@@ -17,7 +17,7 @@ http {
     log_format main '$remote_addr "$request" $status';
     access_log /dev/stdout main;
 
-    upstream collector { server clickstack:4318; }
+    upstream collector { server otel-collector:4318; }
 
     server {
         listen 4319;

dev/docker-compose.yml

@@ -121,51 +121,58 @@ services:
       - main_ai_network
       - prod_net
 
-  # ClickStack (HyperDX all-in-one) — observability UI (logs/traces/metrics)
-  # NB! Port 8124 on ClickHouse'i HTTP; ClickStack UI jookseb pordil 18080.
+  # ClickStack (ametlik "Docker Compose" arhitektuur): HyperDX app-only + eraldi Mongo.
+  # (Varem: all-in-one, mis dubleeris ClickHouse'i + Mongo.) Port 8124 = ClickHouse HTTP; UI = 18080.
   clickstack:
-    image: docker.hyperdx.io/hyperdx/hyperdx-all-in-one:latest
+    image: docker.hyperdx.io/hyperdx/hyperdx:2.37.0
     container_name: clickstack
     restart: unless-stopped
     environment:
       - TZ=Europe/Tallinn
       - FRONTEND_URL=http://100.87.1.24:18080
-      - SERVER_URL=http://100.87.1.24:18080
+      - SERVER_URL=http://127.0.0.1:8000
       - HYPERDX_APP_URL=http://100.87.1.24:18080
       - HYPERDX_APP_PORT=8080
       - HYPERDX_API_PORT=8000
       - HYPERDX_API_KEY=02a19ecf-f62e-4e17-819f-3ee1498d69f3
+      - MONGO_URI=mongodb://clickstack-db:27017/hyperdx
+      - OPAMP_PORT=4320
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+      - OTEL_SERVICE_NAME=hdx-oss-app
       - NEXT_PUBLIC_IS_LOCAL_MODE=false
       # Brauseri (session replay) collector-URL peab olema kliendist kättesaadav (mitte localhost).
       - NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT=http://100.87.1.24:4318
       - EXPRESS_SESSION_SECRET=dev_session_secret_2026
-      - CLICKHOUSE_ENDPOINT=tcp://clickhouse:9000?dial_timeout=10s
-      - CLICKHOUSE_USER=default
-      - CLICKHOUSE_PASSWORD=dev_clickhouse_2026
-      - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
       - 'DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://clickhouse:8123","username":"default","password":"dev_clickhouse_2026"}]'
-    command: ["sh", "/clickstack-entry.sh"]
     ports:
       - "127.0.0.1:18080:8080"
       - "100.87.1.24:18080:8080"   # HyperDX UI
-      - "127.0.0.1:4317:4317"
-      - "100.87.1.24:4317:4317"
-      - "127.0.0.1:4318:4318"
-      - "100.87.1.24:4318:4318"
     volumes:
-      - ./data/clickstack:/data
-      - ./clickstack-entry.sh:/clickstack-entry.sh:ro
       - ./patches/4719-70b065addcfc053d.js:/app/packages/app/packages/app/.next/static/chunks/4719-70b065addcfc053d.js:ro
       - ./patches/pages/clickhouse-46029ebbad38b40a.js:/app/packages/app/packages/app/.next/static/chunks/pages/clickhouse-46029ebbad38b40a.js:ro
       - ./patches/4751-53b929fca5a575a9.js:/app/packages/app/packages/app/.next/static/chunks/4751-53b929fca5a575a9.js:ro
-    # ClickHouse peab olema terve (native 9000), muidu collector'i seed ebaõnnestub
-    # ja OTLP sureb (metrikad/logid/trace'id seisma) -> vale "Teenus maas" alert.
+    # ClickHouse peab olema terve (native 9000), muidu collector'i seed ebaõnnestub.
     depends_on:
+      clickstack-db:
+        condition: service_started
       clickhouse:
         condition: service_healthy
     networks:
       - main_ai_network
 
+  # Mongo: HyperDX rakenduse olek (sources/dashboards/alerts/users). Migreeritud all-in-one'ist.
+  clickstack-db:
+    image: mongo:5.0.32-focal
+    container_name: clickstack-db
+    restart: unless-stopped
+    mem_limit: 512m
+    environment:
+      - TZ=Europe/Tallinn
+    volumes:
+      - ./data/mongo:/data/db
+    networks:
+      - main_ai_network
+
   # Reverse-proxy pordil 8124: /clickstack/ -> ClickHouse staatiline UI,
   # backendi teed -> HyperDX, kõik muu -> ClickHouse (nii jääb 8124 DBeaver-ile tööle)
   clickstack-proxy:
@@ -221,23 +228,35 @@ services:
     networks:
       - main_ai_network
 
-  # OTel kollektor: scrape node-exporter + cadvisor -> OTLP -> clickstack
+  # ClickStack ametlik OTel-collector (OpAMP -> clickstack app). Base config app'ist (skeem/rollupid);
+  # custom.config.yaml lisab hosti-meetrikad (node/cadvisor/llamacpp) + konteinerilogid.
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:latest
+    image: docker.clickhouse.com/clickhouse/clickstack-otel-collector:2
     container_name: otel-collector
     restart: unless-stopped
     mem_limit: 512m
     user: "0:0"   # root: vaja lugeda /var/lib/docker/containers/*-json.log (710 root:root)
-    command: ["--config=/etc/otel/config.yaml"]
     environment:
-      - HYPERDX_INGEST_KEY=${HYPERDX_INGEST_KEY}
+      - TZ=Europe/Tallinn
+      - CLICKHOUSE_ENDPOINT=tcp://clickhouse:9000?dial_timeout=10s
+      - CLICKHOUSE_USER=default
+      - CLICKHOUSE_PASSWORD=dev_clickhouse_2026
+      - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default
+      - OPAMP_SERVER_URL=http://clickstack:4320
+      - CUSTOM_OTELCOL_CONFIG_FILE=/etc/otelcol-contrib/custom.config.yaml
     extra_hosts:
       - "host.docker.internal:host-gateway"
     volumes:
-      - ./config/otel-collector/config.yaml:/etc/otel/config.yaml:ro
+      - ./config/otel-collector/custom.config.yaml:/etc/otelcol-contrib/custom.config.yaml:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+    ports:
+      - "127.0.0.1:4317:4317"
+      - "100.87.1.24:4317:4317"
+      - "127.0.0.1:4318:4318"
+      - "100.87.1.24:4318:4318"
     depends_on:
       - clickstack
+      - clickhouse
       - node-exporter
       - cadvisor
     networks:
@@ -259,7 +278,7 @@ services:
     networks:
       - main_ai_network
 
-  # Rate-limiter HyperDX OTLP ingest'ile (otel.odamus.com). Edge Caddy -> see -> clickstack:4318.
+  # Rate-limiter HyperDX OTLP ingest'ile (otel.odamus.com). Edge Caddy -> see -> otel-collector:4318.
   # nginx limit_req (Caddy standardbuildis rate_limit plugin't pole).
   otel-ratelimit:
     image: nginx:alpine
@@ -273,7 +292,7 @@ services:
     ports:
       - "100.87.1.24:4319:4319"
     depends_on:
-      - clickstack
+      - otel-collector
     networks:
       - main_ai_network