huanggze
huanggze 我是各组件都正常,node监控正常,只有pod监控无数据。
查看prometheus-k8s里的config-reloader有错误日志如下:
level=info ts=2023-11-16T08:46:50.920015191Z caller=main.go:111 msg=“Starting prometheus-config-reloader” version=“(version=0.55.1, branch=refs/tags/v0.55.1, revision=08c846115c67195bc821018168040db6f3e236e3)”
level=info ts=2023-11-16T08:46:50.920115281Z caller=main.go:112 build_context=“(go=go1.17.7, user=Action-Run-ID-2045821452, date=20220326-21:47:32)”
level=info ts=2023-11-16T08:46:50.920557552Z caller=main.go:149 msg=“Starting web server for metrics” listen=:8080
level=error ts=2023-11-16T08:46:50.923734874Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:46:55.923670812Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:00.92373893Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:05.923665708Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:10.928249582Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:15.925540262Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:20.924606563Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:25.927797205Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:30.927690128Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:35.927703602Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:40.92798161Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:45.927804952Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=error ts=2023-11-16T08:47:50.927532979Z caller=runutil.go:101 msg=“function failed. Retrying in next tick” err="trigger reload: reload request failed: Post \“http://localhost:9090/-/reload\”: dial tcp 127.0.0.1:9090: connect: connection refused"
level=info ts=2023-11-16T08:47:59.569229489Z caller=reloader.go:373 msg=“Reload triggered” cfg_in=/etc/prometheus/config/prometheus.yaml.gz cfg_out=/etc/prometheus/config_out/prometheus.env.yaml watched_dirs=/etc/prometheus/rules/prometheus-k8s-rulefiles-0
level=info ts=2023-11-16T08:47:59.569306908Z caller=reloader.go:235 msg=“started watching config file and directories for changes” cfg=/etc/prometheus/config/prometheus.yaml.gz out=/etc/prometheus/config_out/prometheus.env.yaml dirs=/etc/prometheus/rules/prometheus-k8s-rulefiles-0