//Deploy cincinnati pod using dist/openshift/cinci-with-mh-deployment.yaml and app-sre/cincinnati:7bb64d5 image passed, but I don't see signatures in the metadata-helper container. Also curl the endpoint shows file not exist

# oc get pod
NAME                          READY   STATUS    RESTARTS   AGE
cincinnati-68c44bbf45-x2jzq   3/3     Running   0          48m

# oc logs cincinnati-68c44bbf45-x2jzq cincinnati-metadata-helper
[2023-08-17T03:33:11Z INFO  metadata_helper] application settings:
    AppSettings {
        verbosity: Debug,
        address: 0.0.0.0,
        port: 8082,
        path_prefix: "/api/upgrades_info",
        status_address: 0.0.0.0,
        status_port: 9082,
        signatures_dir: "/tmp/cincinnati/graph-data/signatures",
        tracing_endpoint: None,
        backlog: 10,
        max_connections: 10,
        max_connection_rate: 64,
        keep_alive: None,
        client_timeout: 5s,
    }

# oc rsh -c cincinnati-metadata-helper cincinnati-68c44bbf45-x2jzq
sh-4.4$ ls /tmp/cincinnati/graph-data/signatures
ls: cannot access '/tmp/cincinnati/graph-data/signatures': No such file or directory
sh-4.4$ exit

# curl -k https://cincinnati-cincinnati.apps.yanyang-0817a.qe.gcp.devcluster.openshift.com/api/upgrades_info/signatures/sha256=caf073ce29232978c331d421c06ca5c2736ce5461962775fdd760b05fb2496a0/signature-1
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   145  100   145    0     0    454      0 --:--:-- --:--:-- --:--:--   457
{"kind":"does_not_exist","value":"does not exist: signature sha256=caf073ce29232978c331d421c06ca5c2736ce5461962775fdd760b05fb2496a0/signature-1"}
 

app-sre/cincinnati:7bb64d5

Always

1. Deploy cincinnati by using the script and upstream graph-data repo with signature enabled https://github.com/shellyyang1989/cincinnati-graph-data/tree/test-signature
# cat ~/work/cincinnati/hack/deploy_cincinnati.sh 
#!/usr/bin/env bash

set -o nounset
set -o errexit
set -o pipefail

echo -e "\nThis Cincinnati deployment script is just an example; it should not be used in production.\n" >&2

export IMAGE_TAG=7bb64d5

# Create a new namespace/project for Cincinnati
oc new-project cincinnati

# Create a dummy secret as a workaround to not having real secrets
oc create secret generic cincinnati-credentials --from-literal="foo=bar"


# Apply oc template
oc new-app -f dist/openshift/cinci-with-mh-deployment.yaml \
  -p IMAGE_TAG=${IMAGE_TAG}\
  -p GB_PAUSE_SECS=300 \
  -p GB_PLUGIN_SETTINGS="$(cat <<-EOF
      [[plugin_settings]]
      name = "release-scrape-dockerv2"
      repository = "openshift-release-dev/ocp-release"
      fetch_concurrency = 16

      [[plugin_settings]]
      name = "github-secondary-metadata-scrape"
      github_org = "shellyyang1989"
      github_repo = "cincinnati-graph-data"
      reference_branch = "test-signature"
      output_directory = "/tmp/cincinnati-graph-data"

      [[plugin_settings]]
      name = "openshift-secondary-metadata-parse"

      [[plugin_settings]]
      name = "edge-add-remove"
EOF
)" \
  -p ENVIRONMENT_SECRETS="{}" \
  ;

2. Go to the metadata-helper container and check the signatures, but no such directory found
# oc get pod
NAME                          READY   STATUS    RESTARTS   AGE
cincinnati-68c44bbf45-x2jzq   3/3     Running   0          58m

# oc rsh -c cincinnati-metadata-helper cincinnati-68c44bbf45-x2jzq
sh-4.4$ ls /tmp/cincinnati/graph-data/
sh-4.4$ ls /tmp/cincinnati/graph-data/signatures
ls: cannot access '/tmp/cincinnati/graph-data/signatures': No such file or directory
sh-4.4$ 

3.

Metadata-helper container doesn't serve signature as expected
 

Metadata-helper container works
 

Dump cincinnati pod:

# cat cincinnati.yaml 
apiVersion: v1
kind: Pod
metadata:
  annotations:
    k8s.v1.cni.cncf.io/network-status: |-
      [{
          "name": "openshift-sdn",
          "interface": "eth0",
          "ips": [
              "10.131.0.17"
          ],
          "default": true,
          "dns": {}
      }]
    openshift.io/generated-by: OpenShiftNewApp
    openshift.io/scc: restricted-v2
    seccomp.security.alpha.kubernetes.io/pod: runtime/default
  creationTimestamp: "2023-08-17T03:32:55Z"
  generateName: cincinnati-68c44bbf45-
  labels:
    app: cincinnati
    pod-template-hash: 68c44bbf45
  name: cincinnati-68c44bbf45-x2jzq
  namespace: cincinnati
  ownerReferences:
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: ReplicaSet
    name: cincinnati-68c44bbf45
    uid: c9afea1d-1a16-49ad-8664-02c1674210bf
  resourceVersion: "38957"
  uid: 4af7f9f5-8fec-4a5b-b6eb-e212842708e6
spec:
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - podAffinityTerm:
          labelSelector:
            matchExpressions:
            - key: app
              operator: In
              values:
              - cincinnati
          topologyKey: kubernetes.io/hostname
        weight: 100
  containers:
  - args:
    - -c
    - /etc/configs/gb.toml
    command:
    - /usr/bin/graph-builder
    env:
    - name: RUST_BACKTRACE
      valueFrom:
        configMapKeyRef:
          key: gb.rust_backtrace
          name: cincinnati
    envFrom:
    - configMapRef:
        name: environment-secrets
    image: quay.io/app-sre/cincinnati:7bb64d5
    imagePullPolicy: Always
    livenessProbe:
      failureThreshold: 3
      httpGet:
        path: /liveness
        port: 9080
        scheme: HTTP
      initialDelaySeconds: 150
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    name: cincinnati-graph-builder
    ports:
    - containerPort: 8080
      name: graph-builder
      protocol: TCP
    - containerPort: 8090
      name: gb-public
      protocol: TCP
    - containerPort: 9080
      name: status-gb
      protocol: TCP
    readinessProbe:
      failureThreshold: 3
      httpGet:
        path: /readiness
        port: 9080
        scheme: HTTP
      initialDelaySeconds: 150
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    resources:
      limits:
        cpu: 750m
        memory: 768Mi
      requests:
        cpu: 350m
        memory: 128Mi
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 1000680000
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /tmp/cincinnati/graph-data
      name: cincinnati-graph-data
    - mountPath: /etc/secrets
      name: secrets
      readOnly: true
    - mountPath: /etc/configs
      name: configs
      readOnly: true
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-6tcr5
      readOnly: true
  - args:
    - -c
    - /etc/configs/mh.toml
    command:
    - /usr/bin/metadata-helper
    env:
    - name: RUST_BACKTRACE
      valueFrom:
        configMapKeyRef:
          key: gb.rust_backtrace
          name: cincinnati
    envFrom:
    - configMapRef:
        name: environment-secrets
    image: quay.io/app-sre/cincinnati:7bb64d5
    imagePullPolicy: Always
    livenessProbe:
      failureThreshold: 3
      httpGet:
        path: /livez
        port: 9082
        scheme: HTTP
      initialDelaySeconds: 150
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    name: cincinnati-metadata-helper
    ports:
    - containerPort: 8082
      name: metadata-helper
      protocol: TCP
    - containerPort: 9082
      name: status-mh
      protocol: TCP
    readinessProbe:
      failureThreshold: 3
      httpGet:
        path: /readyz
        port: 9082
        scheme: HTTP
      initialDelaySeconds: 150
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    resources:
      limits:
        cpu: 750m
        memory: 768Mi
      requests:
        cpu: 350m
        memory: 128Mi
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 1000680000
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /tmp/cincinnati/graph-data
      name: cincinnati-graph-data
    - mountPath: /etc/secrets
      name: secrets
      readOnly: true
    - mountPath: /etc/configs
      name: configs
      readOnly: true
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-6tcr5
      readOnly: true
  - args:
    - -$(PE_LOG_VERBOSITY)
    - --service.address
    - $(ADDRESS)
    - --service.mandatory_client_parameters
    - $(PE_MANDATORY_CLIENT_PARAMETERS)
    - --service.path_prefix
    - /api/upgrades_info
    - --service.port
    - "8081"
    - --status.address
    - $(PE_STATUS_ADDRESS)
    - --status.port
    - "9081"
    - --upstream.cincinnati.url
    - $(UPSTREAM)
    command:
    - /usr/bin/policy-engine
    env:
    - name: ADDRESS
      valueFrom:
        configMapKeyRef:
          key: pe.address
          name: cincinnati
    - name: PE_STATUS_ADDRESS
      valueFrom:
        configMapKeyRef:
          key: pe.status.address
          name: cincinnati
    - name: UPSTREAM
      valueFrom:
        configMapKeyRef:
          key: pe.upstream
          name: cincinnati
    - name: PE_LOG_VERBOSITY
      valueFrom:
        configMapKeyRef:
          key: pe.log.verbosity
          name: cincinnati
    - name: PE_MANDATORY_CLIENT_PARAMETERS
      valueFrom:
        configMapKeyRef:
          key: pe.mandatory_client_parameters
          name: cincinnati
    - name: RUST_BACKTRACE
      valueFrom:
        configMapKeyRef:
          key: pe.rust_backtrace
          name: cincinnati
    image: quay.io/app-sre/cincinnati:7bb64d5
    imagePullPolicy: Always
    livenessProbe:
      failureThreshold: 3
      httpGet:
        path: /livez
        port: 9081
        scheme: HTTP
      initialDelaySeconds: 300
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    name: cincinnati-policy-engine
    ports:
    - containerPort: 8081
      name: policy-engine
      protocol: TCP
    - containerPort: 9081
      name: status-pe
      protocol: TCP
    readinessProbe:
      failureThreshold: 3
      httpGet:
        path: /readyz
        port: 9081
        scheme: HTTP
      initialDelaySeconds: 300
      periodSeconds: 30
      successThreshold: 1
      timeoutSeconds: 3
    resources:
      limits:
        cpu: 750m
        memory: 1Gi
      requests:
        cpu: 350m
        memory: 128Mi
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 1000680000
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-6tcr5
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  imagePullSecrets:
  - name: default-dockercfg-sklnm
  nodeName: yanyang-0817a-8cnzd-worker-c-w5fll.c.openshift-qe.internal
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 1000680000
    seLinuxOptions:
      level: s0:c26,c15
    seccompProfile:
      type: RuntimeDefault
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  - effect: NoSchedule
    key: node.kubernetes.io/memory-pressure
    operator: Exists
  volumes:
  - name: secrets
    secret:
      defaultMode: 420
      secretName: cincinnati-credentials
  - configMap:
      defaultMode: 420
      name: cincinnati-configs
    name: configs
  - emptyDir: {}
    name: cincinnati-graph-data
  - name: kube-api-access-6tcr5
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
      - configMap:
          items:
          - key: service-ca.crt
            path: service-ca.crt
          name: openshift-service-ca.crt
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2023-08-17T03:32:55Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2023-08-17T03:38:26Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2023-08-17T03:38:26Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2023-08-17T03:32:55Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: cri-o://dd7e4245ba265c0d8655a7ef3747cf2d7e8ccff4b42a05d7249d0ad0be589454
    image: quay.io/app-sre/cincinnati:7bb64d5
    imageID: quay.io/app-sre/cincinnati@sha256:44896428ad8792c8d1fd409e9d3a577baa5a888f2afb6bc9a76b51d9f9587b6b
    lastState: {}
    name: cincinnati-graph-builder
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-08-17T03:33:10Z"
  - containerID: cri-o://3425dfad0898e3c18991ca21d35eb115fff83163bf6aecdef69da4050fdc7bed
    image: quay.io/app-sre/cincinnati:7bb64d5
    imageID: quay.io/app-sre/cincinnati@sha256:44896428ad8792c8d1fd409e9d3a577baa5a888f2afb6bc9a76b51d9f9587b6b
    lastState: {}
    name: cincinnati-metadata-helper
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-08-17T03:33:11Z"
  - containerID: cri-o://42c4cd3aa691d6567128ee2bd6d8a383cad93e38d16b58123de6a0f86692c81b
    image: quay.io/app-sre/cincinnati:7bb64d5
    imageID: quay.io/app-sre/cincinnati@sha256:44896428ad8792c8d1fd409e9d3a577baa5a888f2afb6bc9a76b51d9f9587b6b
    lastState: {}
    name: cincinnati-policy-engine
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-08-17T03:33:12Z"
  hostIP: 10.0.x.x
  phase: Running
  podIP: 10.131.x.x
  podIPs:
  - ip: 10.131.x.x
  qosClass: Burstable
  startTime: "2023-08-17T03:32:55Z"