ts=2026-02-18T04:43:50.365428448Z caller=receive.go:142 level=info component=receive mode=RouterIngestor msg="running receive"
ts=2026-02-18T04:43:50.365490031Z caller=options.go:29 level=info component=receive protocol=HTTP msg="disabled TLS, key and cert must be set to enable"
ts=2026-02-18T04:43:50.365540385Z caller=factory.go:54 level=info component=receive msg="loading bucket configuration"
ts=2026-02-18T04:43:50.365855847Z caller=receive.go:815 level=info component=receive msg="default tenant data dir not found, creating" defaultTenantID=default-tenant
ts=2026-02-18T04:43:50.366417146Z caller=handler.go:165 level=info component=receive component=receive-handler msg="Starting receive handler with async forward workers" workers=5
ts=2026-02-18T04:43:50.366585122Z caller=receive.go:742 level=info component=receive component=uploader msg="upload enabled, starting initial sync"
ts=2026-02-18T04:43:50.366597985Z caller=receive.go:746 level=info component=receive component=uploader msg="initial sync done"
ts=2026-02-18T04:43:50.366848899Z caller=options.go:29 level=info component=receive protocol=gRPC msg="disabled TLS, key and cert must be set to enable"
ts=2026-02-18T04:43:50.367394911Z caller=receive.go:514 level=info component=receive msg="starting receiver"
ts=2026-02-18T04:43:50.367485878Z caller=handler.go:449 level=info component=receive component=receive-handler msg="Start listening for connections" address=0.0.0.0:19291
ts=2026-02-18T04:43:50.367517979Z caller=intrumentation.go:75 level=info component=receive msg="changing probe status" status=healthy
ts=2026-02-18T04:43:50.36751751Z caller=receive.go:415 level=info component=receive msg="listening for StoreAPI and WritableStoreAPI gRPC" address=0.0.0.0:10901
ts=2026-02-18T04:43:50.367561632Z caller=http.go:72 level=info component=receive service=http/server component=receive msg="listening for requests and metrics" address=0.0.0.0:10902
ts=2026-02-18T04:43:50.367563233Z caller=intrumentation.go:75 level=info component=receive msg="changing probe status" status=healthy
ts=2026-02-18T04:43:50.367590623Z caller=handler.go:467 level=info component=receive component=receive-handler msg="Serving plain HTTP" address=0.0.0.0:19291
ts=2026-02-18T04:43:50.367619367Z caller=grpc.go:167 level=info component=receive service=gRPC/server component=receive msg="listening for serving gRPC" address=0.0.0.0:10901
ts=2026-02-18T04:43:50.367622394Z caller=receive.go:605 level=info component=receive msg="Set up hashring for the given hashring config."
ts=2026-02-18T04:43:50.367671089Z caller=intrumentation.go:67 level=warn component=receive msg="changing probe status" status=not-ready reason="hashring has changed; server is not ready to receive requests"
ts=2026-02-18T04:43:50.367682018Z caller=receive.go:701 level=info component=receive msg="hashring has changed; server is not ready to receive requests"
ts=2026-02-18T04:43:50.367688106Z caller=receive.go:703 level=info component=receive msg="updating storage"
ts=2026-02-18T04:43:50.367680853Z caller=handler.go:87 level=info component=receive service=http/server component=receive caller=tls_config.go:347 time=2026-02-18T04:43:50.367670736Z msg="Listening on" address=[::]:10902
ts=2026-02-18T04:43:50.367732426Z caller=handler.go:87 level=info component=receive service=http/server component=receive caller=tls_config.go:350 time=2026-02-18T04:43:50.367725584Z msg="TLS is disabled." http2=false address=[::]:10902
ts=2026-02-18T04:43:50.367752285Z caller=multitsdb.go:716 level=info component=receive component=multi-tsdb tenant=default-tenant msg="opening TSDB"
ts=2026-02-18T04:43:50.373803803Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=default-tenant caller=head.go:638 time=2026-02-18T04:43:50.373792619Z msg="Replaying on-disk memory mappable chunks if any"
ts=2026-02-18T04:43:50.373869234Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=default-tenant caller=head.go:725 time=2026-02-18T04:43:50.373861147Z msg="On-disk memory mappable chunks replay completed" duration=1.564µs
ts=2026-02-18T04:43:50.373884054Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=default-tenant caller=head.go:733 time=2026-02-18T04:43:50.37387873Z msg="Replaying WAL, this may take a while"
ts=2026-02-18T04:43:50.375001169Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=default-tenant caller=head.go:805 time=2026-02-18T04:43:50.374976805Z msg="WAL segment loaded" segment=0 maxSegment=0
ts=2026-02-18T04:43:50.375036216Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=default-tenant caller=head.go:842 time=2026-02-18T04:43:50.375024316Z msg="WAL replay completed" checkpoint_replay_duration=19.46µs wal_replay_duration=1.110078ms wbl_replay_duration=153ns chunk_snapshot_load_duration=0s mmap_chunk_replay_duration=1.564µs total_replay_duration=1.162546ms
ts=2026-02-18T04:43:50.376662979Z caller=multitsdb.go:781 level=info component=receive component=multi-tsdb tenant=default-tenant msg="TSDB is now ready"
ts=2026-02-18T04:43:50.37673298Z caller=shipper.go:307 level=info component=receive component=multi-tsdb tenant=default-tenant msg="no meta file found, creating empty meta data to write later"
ts=2026-02-18T04:43:50.377505643Z caller=intrumentation.go:56 level=info component=receive msg="changing probe status" status=ready
ts=2026-02-18T04:43:50.37752017Z caller=receive.go:717 level=info component=receive msg="storage started, and server is ready to receive requests"
ts=2026-02-18T04:43:50.623174178Z caller=multitsdb.go:716 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="opening TSDB"
ts=2026-02-18T04:43:50.630666653Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:638 time=2026-02-18T04:43:50.630652452Z msg="Replaying on-disk memory mappable chunks if any"
ts=2026-02-18T04:43:50.630741823Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:725 time=2026-02-18T04:43:50.630732076Z msg="On-disk memory mappable chunks replay completed" duration=1.532µs
ts=2026-02-18T04:43:50.63075853Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:733 time=2026-02-18T04:43:50.630752421Z msg="Replaying WAL, this may take a while"
ts=2026-02-18T04:43:50.63176898Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:805 time=2026-02-18T04:43:50.631752628Z msg="WAL segment loaded" segment=0 maxSegment=0
ts=2026-02-18T04:43:50.631823031Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:842 time=2026-02-18T04:43:50.631809922Z msg="WAL replay completed" checkpoint_replay_duration=29.287µs wal_replay_duration=1.010726ms wbl_replay_duration=208ns chunk_snapshot_load_duration=0s mmap_chunk_replay_duration=1.532µs total_replay_duration=1.077183ms
ts=2026-02-18T04:43:50.633536607Z caller=multitsdb.go:781 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="TSDB is now ready"
ts=2026-02-18T04:43:50.71510381Z caller=handler.go:637 level=error component=receive component=receive-handler tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 err="2 errors: forwarding request to endpoint {observability-thanos-receive-default-1.observability-thanos-receive-default.open-cluster-management-observability.svc.cluster.local:10901 observability-thanos-receive-default-1.observability-thanos-receive-default.open-cluster-management-observability.svc.cluster.local:19391 }: rpc error: code = Internal desc = get appender: TSDB not ready; get appender: TSDB not ready" msg="internal server error"
ts=2026-02-18T04:44:20.368182731Z caller=shipper.go:307 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="no meta file found, creating empty meta data to write later"
ts=2026-02-18T07:50:09.193391832Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T07:50:09.193367721Z msg="write block" mint=1771389744772 maxt=1771394400000 ulid=01KHQVMNVBG2Z4MYPR8WQENW8D duration=189.897286ms ooo=false
ts=2026-02-18T07:50:09.206401755Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T07:50:09.206388579Z msg="Head GC completed" caller=truncateMemory duration=12.105244ms
ts=2026-02-18T07:50:20.390005317Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHQVMNVBG2Z4MYPR8WQENW8D
ts=2026-02-18T08:00:50.368422353Z caller=multitsdb.go:449 level=info component=receive component=multi-tsdb msg="Running pruning job"
ts=2026-02-18T09:08:52.044309757Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T09:08:52.044287333Z msg="write block" mint=1771394424714 maxt=1771401600000 ulid=01KHR04SZK28G1NE9GK8CAEMHM duration=216.656259ms ooo=false
ts=2026-02-18T09:08:52.05742618Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T09:08:52.057412756Z msg="Head GC completed" caller=truncateMemory duration=12.089733ms
ts=2026-02-18T09:09:20.389570664Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHR04SZK28G1NE9GK8CAEMHM
ts=2026-02-18T11:08:52.010757509Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T11:08:52.010734172Z msg="write block" mint=1771401602718 maxt=1771408800000 ulid=01KHR70H6CDHC7CJBJRWPRGE2H duration=222.029396ms ooo=false
ts=2026-02-18T11:08:52.022485672Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T11:08:52.022472573Z msg="Head GC completed" caller=truncateMemory duration=10.664893ms
ts=2026-02-18T11:09:20.389497517Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHR70H6CDHC7CJBJRWPRGE2H
ts=2026-02-18T12:00:50.367980497Z caller=multitsdb.go:449 level=info component=receive component=multi-tsdb msg="Running pruning job"
ts=2026-02-18T13:08:34.078733045Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T13:08:34.07871155Z msg="write block" mint=1771408802718 maxt=1771416000000 ulid=01KHRDVPY1HTYXFW0BRGBW7KDW duration=221.071767ms ooo=false
ts=2026-02-18T13:08:34.090912303Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T13:08:34.090899322Z msg="Head GC completed" caller=truncateMemory duration=10.92868ms
ts=2026-02-18T13:08:34.091281893Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=checkpoint.go:100 time=2026-02-18T13:08:34.09126796Z msg="Creating checkpoint" from_segment=0 to_segment=1 mint=1771416000000
ts=2026-02-18T13:08:34.177159287Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1362 time=2026-02-18T13:08:34.177140065Z msg="WAL checkpoint complete" first=0 last=1 duration=85.936291ms
ts=2026-02-18T13:08:50.389052127Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHRDVPY1HTYXFW0BRGBW7KDW
ts=2026-02-18T15:10:56.246922984Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T15:10:56.246898167Z msg="write block" mint=1771416002718 maxt=1771423200000 ulid=01KHRMVS0YNXBZFWVV8G3K5AQ1 duration=216.442408ms ooo=false
ts=2026-02-18T15:10:56.270485665Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T15:10:56.270460733Z msg="Head GC completed" caller=truncateMemory duration=22.334448ms
ts=2026-02-18T15:11:20.388406739Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHRMVS0YNXBZFWVV8G3K5AQ1
ts=2026-02-18T16:00:50.367973162Z caller=multitsdb.go:449 level=info component=receive component=multi-tsdb msg="Running pruning job"
ts=2026-02-18T17:06:39.411681179Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T17:06:39.411659075Z msg="write block" mint=1771423202718 maxt=1771430400000 ulid=01KHRVFE2SJV785NDYXBNSERK8 duration=7.770048509s ooo=false
ts=2026-02-18T17:06:40.184482643Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T17:06:40.184463063Z msg="Head GC completed" caller=truncateMemory duration=767.409985ms
ts=2026-02-18T17:06:40.184852399Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=checkpoint.go:100 time=2026-02-18T17:06:40.18484351Z msg="Creating checkpoint" from_segment=2 to_segment=4 mint=1771430400000
ts=2026-02-18T17:06:40.289351504Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1362 time=2026-02-18T17:06:40.289330448Z msg="WAL checkpoint complete" first=2 last=4 duration=104.601787ms
ts=2026-02-18T17:06:50.387918285Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHRVFE2SJV785NDYXBNSERK8
ts=2026-02-18T19:07:18.231204012Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=compact.go:591 time=2026-02-18T19:07:18.231161935Z msg="write block" mint=1771430400000 maxt=1771437600000 ulid=01KHS2B2CJCV8KP9NZK92R1750 duration=49.604896801s ooo=false
ts=2026-02-18T19:07:20.466733851Z caller=shipper.go:432 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 msg="upload new block" id=01KHS2B2CJCV8KP9NZK92R1750
ts=2026-02-18T19:07:20.819507512Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1400 time=2026-02-18T19:07:20.819487616Z msg="Head GC completed" caller=truncateMemory duration=2.552284702s
ts=2026-02-18T19:07:20.837612188Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=checkpoint.go:100 time=2026-02-18T19:07:20.837595506Z msg="Creating checkpoint" from_segment=5 to_segment=9 mint=1771437600000
ts=2026-02-18T19:07:32.848292496Z caller=handler.go:87 level=info component=receive component=multi-tsdb tenant=f5e0558d-16cb-499e-8b31-50be09a157b9 caller=head.go:1362 time=2026-02-18T19:07:32.848267568Z msg="WAL checkpoint complete" first=5 last=9 duration=12.010837796s
ts=2026-02-18T20:00:50.367858232Z caller=multitsdb.go:449 level=info component=receive component=multi-tsdb msg="Running pruning job"