Uploaded image for project: 'OpenShift Bugs'
  1. OpenShift Bugs
  2. OCPBUGS-23407

Cloud-event-proxy log shows HOLDOVER and FREERUN state when Boundary Clock master interface goes down

XMLWordPrintable

    • Icon: Bug Bug
    • Resolution: Unresolved
    • Icon: Undefined Undefined
    • None
    • 4.14.z
    • Networking / ptp
    • Moderate
    • No
    • False
    • Hide

      None

      Show
      None
    • 1/2: This was not reproduceable in QE lab after upgrading FW to 4.0. QE need to verify one more time to confirm

      When BC master interface is disabled, cloud-event-consumer log shows HOLDOVER and FREERUN state changes.  State does return to LOCKED, but no state change should occur.

      Environment:

      [kni@registry.ran-vcl01 ptp]$ oc get csv -n openshift-ptp
      NAME                                DISPLAY        VERSION               REPLACES   PHASE
      ptp-operator.v4.14.0-202311092032   PTP Operator   4.14.0-202311092032              Succeeded
      
      [root@cnfde4 /]# uname -r
      5.14.0-284.41.1.rt14.326.el9_2.x86_64
      [kni@registry.ran-vcl01 ptp]$ oc get ptpconfigs -A
      NAMESPACE       NAME         AGE
      openshift-ptp   du-ptp-bc1   42h
      [kni@registry.ran-vcl01 ptp]$ oc get ptpconfigs -A -o yaml 
      apiVersion: v1
      items:
      - apiVersion: ptp.openshift.io/v1
        kind: PtpConfig
        metadata:
          creationTimestamp: "2023-11-15T02:24:43Z"
          generation: 15
          name: du-ptp-bc1
          namespace: openshift-ptp
          resourceVersion: "493105"
          uid: ec9a4777-7af2-4ad5-ba34-a36125183b22
        spec:
          profile:
          - name: bc1
            phc2sysOpts: -s ens4f0 -w -r -m -n 24 -N 8 -R 16
            ptp4lConf: |
              # The interface name is hardware-specific
              [ens4f0]
              masterOnly 0
              [ens4f1]
              masterOnly 1
              [ens4f2]
              masterOnly 1
              [ens4f3]
              masterOnly 1
              [global]
              #
              # Default Data Set
              #
              twoStepFlag 1
              slaveOnly 0
              priority1 128
              priority2 128
              domainNumber 24
              #utc_offset 37
              clockClass 248
              clockAccuracy 0xFE
              offsetScaledLogVariance 0xFFFF
              free_running 0
              freq_est_interval 1
              dscp_event 0
              dscp_general 0
              dataset_comparison G.8275.x
              G.8275.defaultDS.localPriority 128
              #
              # Port Data Set
              #
              logAnnounceInterval -3
              logSyncInterval -4
              logMinDelayReqInterval -4
              logMinPdelayReqInterval -4
              announceReceiptTimeout 3
              syncReceiptTimeout 0
              delayAsymmetry 0
              fault_reset_interval -4
              neighborPropDelayThresh 20000000
              masterOnly 0
              G.8275.portDS.localPriority 128
              #
              # Run time options
              #
              assume_two_step 0
              logging_level 6
              path_trace_enabled 0
              follow_up_info 0
              hybrid_e2e 0
              inhibit_multicast_service 0
              net_sync_monitor 0
              tc_spanning_tree 0
              tx_timestamp_timeout 50
              unicast_listen 0
              unicast_master_table 0
              unicast_req_duration 3600
              use_syslog 1
              verbose 0
              summary_interval 0
              kernel_leap 1
              check_fup_sync 0
              # change clock_class_threshold to 248 for BC workaround
              clock_class_threshold 248
              #
              # Servo Options
              #
              pi_proportional_const 0.0
              pi_integral_const 0.0
              pi_proportional_scale 0.0
              pi_proportional_exponent -0.3
              pi_proportional_norm_max 0.7
              pi_integral_scale 0.0
              pi_integral_exponent 0.4
              pi_integral_norm_max 0.3
              step_threshold 2.0
              first_step_threshold 0.00002
              max_frequency 900000000
              clock_servo pi
              sanity_freq_limit 200000000
              ntpshm_segment 0
              #
              # Transport options
              #
              transportSpecific 0x0
              ptp_dst_mac 01:1B:19:00:00:00
              p2p_dst_mac 01:80:C2:00:00:0E
              udp_ttl 1
              udp6_scope 0x0E
              uds_address /var/run/ptp4l
              #
              # Default interface options
              #
              clock_type BC
              network_transport L2
              delay_mechanism E2E
              time_stamping hardware
              tsproc_mode filter
              delay_filter moving_median
              delay_filter_length 10
              egressLatency 0
              ingressLatency 0
              boundary_clock_jbod 0
              #
              # Clock description
              #
              productDescription ;;
              revisionData ;;
              manufacturerIdentity 00:00:00
              userDescription ;
              timeSource 0xA0
            ptp4lOpts: -2 --summary_interval -4
            ptpSchedulingPolicy: SCHED_FIFO
            ptpSchedulingPriority: 10
            ptpSettings:
              logReduce: "false"
          recommend:
          - match:
            - nodeLabel: node-role.kubernetes.io/master
            priority: 4
            profile: bc1
      kind: List
      metadata:
        resourceVersion: ""

      Steps to reproduce:

      [kni@registry.ran-vcl01 ptp]$ oc exec -it ds/linuxptp-daemon  -n openshift-ptp -c linuxptp-daemon-container -it /bin/bash
      [root@cnfde4 /]# ip link set ens4f1 down

      cloud-event-proxy log after disabling ens4f1:

      $ oc logs -n cloud-events cloud-consumer-deployment-66b8497478-27lzg cloud-event-consumer -f --since=0s
      time="2023-11-16T19:16:43Z" level=info msg="Latency for the event: 1 ms"
      time="2023-11-16T19:16:43Z" level=info msg="received event {\"id\":\"dc592fcb-4f05-469f-8ad4-7b12164cb2a1\",\"type\":\"event.sync.ptp-status.ptp-state-change\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/ptp-status/lock-state\",\"dataContentType\":\"application/json\",\"time\":\"2023-11-16T19:16:43.693895334Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"HOLDOVER\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"-3\"}]}}"
      time="2023-11-16T19:16:43Z" level=info msg="Latency for the event: 1 ms"
      time="2023-11-16T19:16:43Z" level=info msg="received event {\"id\":\"8617f2e1-f688-4c36-900c-5b9fc8f9d200\",\"type\":\"event.sync.sync-status.os-clock-sync-state-change\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/sync-status/os-clock-sync-state\",\"dataContentType\":\"application/json\",\"time\":\"2023-11-16T19:16:43.69405063Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"FREERUN\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"9\"}]}}"
      time="2023-11-16T19:16:43Z" level=info msg="Latency for the event: 0 ms"
      time="2023-11-16T19:16:43Z" level=info msg="received event {\"id\":\"dc592fcb-4f05-469f-8ad4-7b12164cb2a1\",\"type\":\"event.sync.ptp-status.ptp-state-change\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/ptp-status/lock-state\",\"dataContentType\":\"application/json\",\"time\":\"2023-11-16T19:16:43.779667349Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"FREERUN\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"-1e+16\"}]}}"
      time="2023-11-16T19:16:43Z" level=info msg="Latency for the event: 0 ms"
      time="2023-11-16T19:16:43Z" level=info msg="received event {\"id\":\"8617f2e1-f688-4c36-900c-5b9fc8f9d200\",\"type\":\"event.sync.sync-status.os-clock-sync-state-change\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/sync-status/os-clock-sync-state\",\"dataContentType\":\"application/json\",\"time\":\"2023-11-16T19:16:43.801906728Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"LOCKED\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"-10\"}]}}"
      time="2023-11-16T19:16:43Z" level=info msg="Latency for the event: 0 ms"
      time="2023-11-16T19:16:43Z" level=info msg="received event {\"id\":\"dc592fcb-4f05-469f-8ad4-7b12164cb2a1\",\"type\":\"event.sync.ptp-status.ptp-state-change\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/ptp-status/lock-state\",\"dataContentType\":\"application/json\",\"time\":\"2023-11-16T19:16:43.826788756Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"LOCKED\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"-8\"}]}}"
      time="2023-11-16T19:17:25Z" level=error msg="CurrentState:error 400 from url http://127.0.0.1:8089/api/ocloudNotifications/v1/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/gnss-status/gnss-sync-status/CurrentState, {\"error\":\"{\\\"message\\\":\\\"could not find any events for requested resource type /cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/gnss-status/gnss-sync-status\\\"}\\n\"}"
      time="2023-11-16T19:17:25Z" level=debug msg="Got CurrentState: {\"specversion\":\"0.3\",\"id\":\"2e3d23ba-d429-4efb-87a2-a3471321fae7\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/ptp-status/lock-state\",\"type\":\"event.sync.ptp-status.ptp-state-change\",\"subject\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/ptp-status/lock-state\",\"datacontenttype\":\"application/json\",\"time\":\"2023-11-16T19:17:25.64554722Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"LOCKED\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/ens4fx/master\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"3\"}]}} "
      time="2023-11-16T19:17:25Z" level=debug msg="Got CurrentState: {\"specversion\":\"0.3\",\"id\":\"e13398c4-b208-45f4-98a0-21ad16d5a02a\",\"source\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/sync-status/os-clock-sync-state\",\"type\":\"event.sync.sync-status.os-clock-sync-state-change\",\"subject\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/sync/sync-status/os-clock-sync-state\",\"datacontenttype\":\"application/json\",\"time\":\"2023-11-16T19:17:25.647525697Z\",\"data\":{\"version\":\"v1\",\"values\":[{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"notification\",\"valueType\":\"enumeration\",\"value\":\"LOCKED\"},{\"resource\":\"/cluster/node/cnfde4.ptp.lab.eng.bos.redhat.com/CLOCK_REALTIME\",\"dataType\":\"metric\",\"valueType\":\"decimal64.3\",\"value\":\"3\"}]}} "

            aputtur@redhat.com Aneesh Puttur
            bblock@redhat.com Bonnie Block
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated: