component=ingester loop=1 org_id=infrastructure msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: Timeout when reading or writing data, num_chunks: 1, labels: {__stream_shard__=\"1\", k8s_node_name=\"kbhartiosp2-gtrlv-master-0\", kubernetes_host=\"kbhartiosp2-gtrlv-master-0\", log_type=\"infrastructure\", openshift_log_type=\"infrastructure\"}"

level=error ts=2025-11-06T18:50:00.645923306Z caller=log.go:223 msg="error running loki" err="failed to create object client: Timeout when reading or writing data\nerror initialising module: compactor\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/opt/app-root/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:138\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/opt/app-root/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/v3/pkg/loki.(*Loki).Run\n\t/opt/app-root/src/loki/pkg/loki/loki.go:531\nmain.main\n\t/opt/app-root/src/loki/cmd/loki/main.go:129\nruntime.main\n\t/usr/lib/golang/src/runtime/proc.go:283\nruntime.goexit\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1700"

level=error ts=2025-11-06T18:49:27.67935856Z caller=log.go:223 msg="error running loki" err="Timeout when reading or writing data\nerror creating object client\ngithub.com/grafana/loki/v3/pkg/storage.(*LokiStore).chunkClientForPeriod\n\t/opt/app-root/src/loki/pkg/storage/store.go:248\ngithub.com/grafana/loki/v3/pkg/storage.(*LokiStore).init\n\t/opt/app-root/src/loki/pkg/storage/store.go:197\ngithub.com/grafana/loki/v3/pkg/storage.NewStore\n\t/opt/app-root/src/loki/pkg/storage/store.go:189\ngithub.com/grafana/loki/v3/pkg/loki.(*Loki).initStore\n\t/opt/app-root/src/loki/pkg/loki/modules.go:929\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/opt/app-root/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/opt/app-root/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/v3/pkg/loki.(*Loki).Run\n\t/opt/app-root/src/loki/pkg/loki/loki.go:531\nmain.main\n\t/opt/app-root/src/loki/cmd/lok...

common:
  storage:
    swift:
      auth_url: https://rhos-d.infra.prod.upshift.rdu2.redhat.com:13000/v3
      username: ${SWIFT_USERNAME}
      user_domain_name: redhat.com
      user_domain_id: <hidden>
      user_id: <hidden>
      password: ${SWIFT_PASSWORD}
      domain_id: <hidden>
      domain_name: redhat.com
      project_id: <hidden>
      project_name: openshift-qe-jenkins
      project_domain_id: <hidden>
      project_domain_name: redhat.com
      region_name: 
      container_name: logging-loki-74397-kbhartiosp1-nxhnk

$oc get networkpolicy loki-74397-loki-allow-bucket-egress -o yaml | yq -e .spec
egress:
  - ports:
      - port: 13000
        protocol: TCP
podSelector:
  matchExpressions:
    - key: app.kubernetes.io/name
      operator: In
      values:
        - lokistack
    - key: app.kubernetes.io/component
      operator: In
      values:
        - ingester
        - querier
        - index-gateway
        - compactor
        - ruler
policyTypes:
  - Egress

$ oc get lokistack loki-74397 -o yaml | yq -e .spec
limits:
  ....
managementState: Managed
networkPolicies:
  disabled: false
rules:
  enabled: true
  namespaceSelector:
    matchLabels:
      openshift.io/cluster-monitoring: "true"
  selector:
    matchLabels:
      openshift.io/cluster-monitoring: "true"
size: 1x.demo
storage:
  schemas:
    - effectiveDate: "2023-10-15"
      version: v13
  secret:
    name: storage-secret-74397
    type: swift
storageClassName: standard-csi
tenants:
  mode: openshift-logging
  openshift:
    adminGroups: []