Hi All,
I am trying to use prometheus to display data with exemplars to show traceID but I cannot seem to get it to display the exemplars
I used this repo as a base and placed in prometheus and grafana tempo.
#Docker Compose
version: '3.5'
networks:
solace_msg_net:
name: solace_msg_net
external: false
driver: bridge
services:
#Prometheus
prometheus:
container_name: prometheus
image: prom/prometheus
networks:
- solace_msg_net
restart: always
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --enable-feature=exemplar-storage #Enable exemplar storage
volumes:
- ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml
ports:
- "9090:9090"
Tempo
tempo:
image: grafana/tempo:latest
restart: always
command: ["-config.file=/etc/tempo.yaml"]
networks:
- solace_msg_net
container_name: tempo1
hostname: tempo
volumes:
- ./tempo/tempo-config.yaml:/etc/tempo.yaml
ports:
- "3200:3200"
- "4317:4317"
healthcheck:
interval: 5s
retries: 10
test: wget --no-verbose --tries=1 --spider http://localhost:3200/status || exit 1
Collector
otel-collector:
image: ${OTELCOL_IMG}
command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"]
hostname: otelcollector
networks:
- solace_msg_net
volumes:
- ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml
ports:
- "1888:1888" # pprof extension
- "8888:8888" # Prometheus metrics exposed by the collector
- "8889:8889" # Prometheus exporter metrics
- "13133:13133" # health_check extension
- "55679:55679" # zpages extension
solbroker:
image: $PUBSUB_IMAGE
hostname: $PUBSUB_HOSTNAME
networks:
- solace_msg_net
env_file:
- ./solace_config_keys.env
ports:
- "2222:2222"
- "8080:8080"
- "55443:55443"
- "55445:55445"
- "${PUBSUB_PORT_NUMBER}:55555"
- "55556:55556"
- "5671:5671"
- "5672:5672"
- "5550:5550"
- "1943:1943"
- "8008:8008"
shm_size: 2g
ulimits:
memlock: -1
nofile:
soft: 2448
hard: 42192
solace_prometheus_exportor:
image: solacecommunity/solace-prometheus-exporter:latest
container_name: prometheus-exporter
networks:
- solace_msg_net
environment:
- SOLACE_LISTEN_ADDR=0.0.0.0:9628
- SOLACE_SCRAPE_URI=http://solbroker:8080
- SOLACE_USERNAME=admin
- SOLACE_PASSWORD=admin
- SOLACE_TIMEOUT=5s
- SOLACE_DEFAULT_VPN=default
- SOLACE_SSL_VERIFY=false
- SOLACE_LISTEN_TLS=false
ports:
- 9628:9628
grafana:
image: grafana/grafana
container_name: grafana1
hostname: grafana
networks:
- solace_msg_net
depends_on:
tempo:
condition: service_healthy
volumes:
- ./grafana/grafana-bootstrap.ini:/etc/grafana/grafana.ini
- ./grafana/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_DISABLE_LOGIN_FORM=true
ports:
- "3000:3000"
healthcheck:
interval: 5s
retries: 10
test: wget --no-verbose --tries=1 --spider http://localhost:3000 || exit 1
#Otel Collector
receivers:
otlp:
protocols:
grpc:
solace:
broker: [solbroker:5672]
max_unacknowledged: 500
auth:
sasl_plain:
username: trace
password: trace
queue: queue://#telemetry-trace
tls:
insecure: true
insecure_skip_verify: true
processors:
memory_limiter:
check_interval: 1s
limit_mib: 1000
spike_limit_mib: 500
batch:
exporters:
prometheus:
endpoint: localhost:9090
enable_open_metrics: true
otlp:
endpoint: tempo:4317
tls:
insecure: true
service:
pipelines:
metrics:
receivers: [otlp]
processors: [batch]
exporters: [prometheus]
traces:
receivers: [solace, otlp]
processors: [batch, memory_limiter]
exporters: [otlp]
#Prometheus.yaml
global:
scrape_interval: 10s
evaluation_interval: 10s
scrape_configs:
job_name: 'tempo'
static_configs:targets: ['tempo:3200']job_name: 'otel-collector'
metrics_path: /metrics
static_configs:targets: ['otel-collector:8888']job_name: 'solace'
metrics_path: /solace-det
static_configs:targets: ['solace_prometheus_exportor:9628']
I am wondering if I missed out something.
Thanks.
