wanjunlei 这个是其中一个pod的日志
{“error”:{“root_cause”:[{“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4340513, max_coordinating_and_primary_bytes=322122547]”}],“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4340513, max_coordinating_and_primary_bytes=322122547]”},“status”:429}
[2023/01/13 10:54:17] [ warn] [engine] failed to flush chunk ‘17-1673607251.835139423.flb’, retry in 11 seconds: task_id=1, input=systemd.2 > output=es.0 (out_id=0)
[2023/01/13 10:54:17] [error] [output:es:es.0] HTTP status=429 URI=/_bulk, response:
{“error”:{“root_cause”:[{“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4338731, max_coordinating_and_primary_bytes=322122547]”}],“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4338731, max_coordinating_and_primary_bytes=322122547]”},“status”:429}
[2023/01/13 10:54:17] [ warn] [engine] failed to flush chunk ‘17-1673607251.664272896.flb’, retry in 9 seconds: task_id=0, input=systemd.2 > output=es.0 (out_id=0)
[2023/01/13 10:54:17] [error] [output:es:es.0] HTTP status=429 URI=/_bulk, response:
{“error”:{“root_cause”:[{“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4341587, max_coordinating_and_primary_bytes=322122547]”}],“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=300086987, replica_bytes=17952278, all_bytes=318039265, coordinating_operation_bytes=4341587, max_coordinating_and_primary_bytes=322122547]”},“status”:429}
[2023/01/13 10:54:17] [ warn] [engine] failed to flush chunk ‘17-1673607252.621261412.flb’, retry in 8 seconds: task_id=5, input=systemd.2 > output=es.0 (out_id=0)
[2023/01/13 10:54:24] [error] [output:es:es.0] HTTP status=429 URI=/_bulk, response:
{“error”:{“root_cause”:[{“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=288825403, replica_bytes=34626716, all_bytes=323452119, coordinating_operation_bytes=4341587, max_coordinating_and_primary_bytes=322122547]”}],“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=288825403, replica_bytes=34626716, all_bytes=323452119, coordinating_operation_bytes=4341587, max_coordinating_and_primary_bytes=322122547]”},“status”:429}
[2023/01/13 10:54:24] [ warn] [engine] chunk ‘17-1673607252.621261412.flb’ cannot be retried: task_id=5, input=systemd.2 > output=es.0
[2023/01/13 10:54:25] [error] [output:es:es.0] HTTP status=429 URI=/_bulk, response:
{“error”:{“root_cause”:[{“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=287230797, replica_bytes=32292331, all_bytes=319523128, coordinating_operation_bytes=4338731, max_coordinating_and_primary_bytes=322122547]”}],“type”:“es_rejected_execution_exception”,“reason”:“rejected execution of coordinating operation [coordinating_and_primary_bytes=287230797, replica_bytes=32292331, all_bytes=319523128, coordinating_operation_bytes=4338731, max_coordinating_and_primary_bytes=322122547]”},“status”:429}
[2023/01/13 10:54:25] [ warn] [engine] chunk ‘17-1673607251.664272896.flb’ cannot be retried: task_id=0, input=systemd.2 > output=es.0
[2023/01/13 10:54:41] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:54:41] [error] [output:es:es.0] could not pack/validate JSON response
{“took”:19135,“errors”:true,“items”:[{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“9a53d7db-a11b-e900-1257-5308496b9fa4”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137969,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“e582829b-9b47-ed58-7e94-99b79b020d0c”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137970,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“20e6722c-6233-aa47-1746-c388bdad9510”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137971,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“fa5d8fce-4cd4-ce45-a226-b839a7e3f62e”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137972,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“3f62d996-1d20-b3df-006b-8575859ca7ea”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137973,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“dc03f40e-fb37-fbbc-cefe-8148a358887a”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137974,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“c47d7a1a-fd4a-a689-3897-db3fd75871d9”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137975,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“a1da67ae-eee3-a6d6-2e50-89e1f1b625f5”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137976,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“2ffe86c4-5949-5e81-1af5-1d24253e00a0”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137977,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“55afe9d4-ac7b-0884-1848-46521b0a2fc5”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137978,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“5557cbd9-31b8-990d-ca4c-fae691bf65aa”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137979,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“e8914c52-18af-c5e1-f53a-6e7f3ae1ac0c”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137980,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“ab492b4f-de3a-389a-29d1-24ee40e4c8cb”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137981,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“f3f61f6b-21ac-f551-1624-16ae91de66b5”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137982,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“a0d68c2f-b199-682e-d453-921ed0533b85”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137983,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“33ad7988-e765-c9e1-497b-1bec067da9c5”,“version”:1,“result”:“created”,“shards”:{“total”:2,“successful”:2,“failed”:0},“seq_no”:137984,“primary_term”:3,“status”:201}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“ac59d8de-a64f-84cd-8979-f13a3149dbcd”,“_version”:1,“result”:"created
[2023/01/13 10:54:41] [ warn] [engine] failed to flush chunk ‘17-1673607252.30882341.flb’, retry in 10 seconds: task_id=2, input=systemd.2 > output=es.0 (out_id=0)
[2023/01/13 10:54:42] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:54:42] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:54:44] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:54:50] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:55:26] [ warn] [http_client] cannot increase buffer: current=512000 requested=544768 max=512000
[2023/01/13 10:55:26] [error] [output:es:es.0] could not pack/validate JSON response
{“took”:28284,“errors”:true,“items”:[{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“9a53d7db-a11b-e900-1257-5308496b9fa4”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[9a53d7db-a11b-e900-1257-5308496b9fa4]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“e582829b-9b47-ed58-7e94-99b79b020d0c”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[e582829b-9b47-ed58-7e94-99b79b020d0c]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“20e6722c-6233-aa47-1746-c388bdad9510”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[20e6722c-6233-aa47-1746-c388bdad9510]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“fa5d8fce-4cd4-ce45-a226-b839a7e3f62e”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[fa5d8fce-4cd4-ce45-a226-b839a7e3f62e]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“3f62d996-1d20-b3df-006b-8575859ca7ea”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[3f62d996-1d20-b3df-006b-8575859ca7ea]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“dc03f40e-fb37-fbbc-cefe-8148a358887a”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[dc03f40e-fb37-fbbc-cefe-8148a358887a]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“c47d7a1a-fd4a-a689-3897-db3fd75871d9”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[c47d7a1a-fd4a-a689-3897-db3fd75871d9]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“a1da67ae-eee3-a6d6-2e50-89e1f1b625f5”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[a1da67ae-eee3-a6d6-2e50-89e1f1b625f5]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“2ffe86c4-5949-5e81-1af5-1d24253e00a0”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[2ffe86c4-5949-5e81-1af5-1d24253e00a0]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“55afe9d4-ac7b-0884-1848-46521b0a2fc5”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”,“reason”:“[55afe9d4-ac7b-0884-1848-46521b0a2fc5]: version conflict, document already exists (current version [1])”,“index_uuid”:“Jp9hHFS_TdqnHzk4h1y_zw”,“shard”:“0”,“index”:“ks-logstash-log-2023.01.08”}}},{“create”:{“index”:“ks-logstash-log-2023.01.08”,“type”:“doc”,“id”:“5557cbd9-31b8-990d-ca4c-fae691bf65aa”,“status”:409,“error”:{“type”:“version_conflict_engine_exception”
[2023/01/13 10:55:26] [ warn] [engine] chunk ‘17-1673607252.30882341.flb’ cannot be retried: task_id=2, input=systemd.2 > output=es.0