Uploaded image for project: 'OpenShift Bugs'
  1. OpenShift Bugs
  2. OCPBUGS-37719

Sometimes ptp4l offset spikes while deleting and re-creating a test application consisting of multiple pods

XMLWordPrintable

    • Important
    • None
    • False
    • Hide

      None

      Show
      None

      Description of problem:

      Sometimes ptp4l offset spikes while deleting and re-creating a test application consisting of multiple pods:
      
      log snippet(attaching full log in must-gather):
      
      2024-07-30T12:46:18.340223328Z phc2sys[1923.879]: [ptp4l.0.config] CLOCK_REALTIME phc offset         1 s2 freq  +19777 delay    537
      2024-07-30T12:46:18.402822673Z phc2sys[1923.942]: [ptp4l.0.config] CLOCK_REALTIME phc offset         0 s2 freq  +19776 delay    533
      2024-07-30T12:46:18.465414816Z phc2sys[1924.004]: [ptp4l.0.config] CLOCK_REALTIME phc offset         4 s2 freq  +19780 delay    541
      2024-07-30T12:46:18.528076756Z phc2sys[1924.067]: [ptp4l.0.config] CLOCK_REALTIME phc offset         2 s2 freq  +19779 delay    531
      2024-07-30T12:46:18.590654654Z phc2sys[1924.129]: [ptp4l.0.config] CLOCK_REALTIME phc offset 999999999 s2 freq +100000000 delay    534
      2024-07-30T12:46:18.653251033Z phc2sys[1924.192]: [ptp4l.0.config] CLOCK_REALTIME phc offset 993048737 s2 freq +100000000 delay    481
      2024-07-30T12:46:18.715911362Z phc2sys[1924.255]: [ptp4l.0.config] CLOCK_REALTIME phc offset 986095430 s2 freq +100000000 delay    487
      2024-07-30T12:46:18.778437714Z phc2sys[1924.317]: [ptp4l.0.config] CLOCK_REALTIME phc offset 979141954 s2 freq +100000000 delay    482
          

      Version-Release number of selected component (if applicable):

      4.14.33
      ptp-operator.v4.14.0-202407021509    

      How reproducible:

      sporadic    

      Steps to Reproduce:

          1. SNO with dual boundary clock configuration
      
      apiVersion: v1
      items:
      - apiVersion: ptp.openshift.io/v1
        kind: PtpConfig
        metadata:
          creationTimestamp: "2024-07-17T11:23:58Z"
          generation: 5
          name: boundary0
          namespace: openshift-ptp
          resourceVersion: "2938558"
          uid: 6d40048f-ccc1-40c0-9fa5-79a5c1420a1b
        spec:
          profile:
          - name: boundary0
            phc2sysOpts: -a -r -m -n 24 -N 8 -R 16
            ptp4lConf: |
              [ens1f3]
              masterOnly 0
              [ens1f0]
              masterOnly 1
              [ens1f1]
              masterOnly 1
              [ens1f2]
              masterOnly 1
              [global]
              #
              # Default Data Set
              #
              twoStepFlag 1
              slaveOnly 0
              priority1 128
              priority2 128
              domainNumber 24
              #utc_offset 37
              clockClass 248
              clockAccuracy 0xFE
              offsetScaledLogVariance 0xFFFF
              free_running 0
              freq_est_interval 1
              dscp_event 0
              dscp_general 0
              dataset_comparison G.8275.x
              G.8275.defaultDS.localPriority 128
              #
              # Port Data Set
              #
              logAnnounceInterval -3
              logSyncInterval -4
              logMinDelayReqInterval -4
              logMinPdelayReqInterval -4
              announceReceiptTimeout 3
              syncReceiptTimeout 0
              delayAsymmetry 0
              fault_reset_interval -4
              neighborPropDelayThresh 20000000
              masterOnly 0
              G.8275.portDS.localPriority 128
              #
              # Run time options
              #
              assume_two_step 0
              logging_level 6
              path_trace_enabled 0
              follow_up_info 0
              hybrid_e2e 0
              inhibit_multicast_service 0
              net_sync_monitor 0
              tc_spanning_tree 0
              tx_timestamp_timeout 50
              unicast_listen 0
              unicast_master_table 0
              unicast_req_duration 3600
              use_syslog 1
              verbose 0
              summary_interval 0
              kernel_leap 1
              check_fup_sync 0
              clock_class_threshold 135
              #
              # Servo Options
              #
              pi_proportional_const 0.0
              pi_integral_const 0.0
              pi_proportional_scale 0.0
              pi_proportional_exponent -0.3
              pi_proportional_norm_max 0.7
              pi_integral_scale 0.0
              pi_integral_exponent 0.4
              pi_integral_norm_max 0.3
              step_threshold 2.0
              first_step_threshold 0.00002
              max_frequency 900000000
              clock_servo pi
              sanity_freq_limit 200000000
              ntpshm_segment 0
              #
              # Transport options
              #
              transportSpecific 0x0
              ptp_dst_mac 01:1B:19:00:00:00
              p2p_dst_mac 01:80:C2:00:00:0E
              udp_ttl 1
              udp6_scope 0x0E
              uds_address /var/run/ptp4l
              #
              # Default interface options
              #
              clock_type BC
              network_transport L2
              delay_mechanism E2E
              time_stamping hardware
              tsproc_mode filter
              delay_filter moving_median
              delay_filter_length 10
              egressLatency 0
              ingressLatency 0
              boundary_clock_jbod 0
              #
              # Clock description
              #
              productDescription ;;
              revisionData ;;
              manufacturerIdentity 00:00:00
              userDescription ;
              timeSource 0xA0
            ptp4lOpts: "-2"
            ptpSchedulingPolicy: SCHED_FIFO
            ptpSchedulingPriority: 10
            ptpSettings:
              logReduce: "true"
          recommend:
          - match:
            - nodeLabel: node-role.kubernetes.io/master
            priority: 4
            profile: boundary0
      - apiVersion: ptp.openshift.io/v1
        kind: PtpConfig
        metadata:
          creationTimestamp: "2024-07-17T11:23:58Z"
          generation: 7
          name: boundary1
          namespace: openshift-ptp
          resourceVersion: "2940211"
          uid: a8b66a82-6abe-4b13-bf5b-3b7e97f6f29b
        spec:
          profile:
          - name: boundary1
            phc2sysOpts: ""
            ptp4lConf: |
              [ens2f3]
              masterOnly 0
              [ens2f0]
              masterOnly 1
              [ens2f1]
              masterOnly 1
              [ens2f2]
              masterOnly 1
              [global]
              #
              # Default Data Set
              #
              twoStepFlag 1
              slaveOnly 0
              priority1 128
              priority2 128
              domainNumber 24
              #utc_offset 37
              clockClass 248
              clockAccuracy 0xFE
              offsetScaledLogVariance 0xFFFF
              free_running 0
              freq_est_interval 1
              dscp_event 0
              dscp_general 0
              dataset_comparison G.8275.x
              G.8275.defaultDS.localPriority 128
              #
              # Port Data Set
              #
              logAnnounceInterval -3
              logSyncInterval -4
              logMinDelayReqInterval -4
              logMinPdelayReqInterval -4
              announceReceiptTimeout 3
              syncReceiptTimeout 0
              delayAsymmetry 0
              fault_reset_interval -4
              neighborPropDelayThresh 20000000
              masterOnly 0
              G.8275.portDS.localPriority 128
              #
              # Run time options
              #
              assume_two_step 0
              logging_level 6
              path_trace_enabled 0
              follow_up_info 0
              hybrid_e2e 0
              inhibit_multicast_service 0
              net_sync_monitor 0
              tc_spanning_tree 0
              tx_timestamp_timeout 50
              unicast_listen 0
              unicast_master_table 0
              unicast_req_duration 3600
              use_syslog 1
              verbose 0
              summary_interval 0
              kernel_leap 1
              check_fup_sync 0
              clock_class_threshold 135
              #
              # Servo Options
              #
              pi_proportional_const 0.0
              pi_integral_const 0.0
              pi_proportional_scale 0.0
              pi_proportional_exponent -0.3
              pi_proportional_norm_max 0.7
              pi_integral_scale 0.0
              pi_integral_exponent 0.4
              pi_integral_norm_max 0.3
              step_threshold 2.0
              first_step_threshold 0.00002
              max_frequency 900000000
              clock_servo pi
              sanity_freq_limit 200000000
              ntpshm_segment 0
              #
              # Transport options
              #
              transportSpecific 0x0
              ptp_dst_mac 01:1B:19:00:00:00
              p2p_dst_mac 01:80:C2:00:00:0E
              udp_ttl 1
              udp6_scope 0x0E
              uds_address /var/run/ptp4l
              #
              # Default interface options
              #
              clock_type BC
              network_transport L2
              delay_mechanism E2E
              time_stamping hardware
              tsproc_mode filter
              delay_filter moving_median
              delay_filter_length 10
              egressLatency 0
              ingressLatency 0
              boundary_clock_jbod 0
              #
              # Clock description
              #
              productDescription ;;
              revisionData ;;
              manufacturerIdentity 00:00:00
              userDescription ;
              timeSource 0xA0
            ptp4lOpts: "-2"
            ptpSchedulingPolicy: SCHED_FIFO
            ptpSchedulingPriority: 10
            ptpSettings:
              logReduce: "true"
          recommend:
          - match:
            - nodeLabel: node-role.kubernetes.io/master
            priority: 4
            profile: boundary1
      kind: List
      metadata:
        resourceVersion: ""
      
      
          2. Delete and re-create a test workload multiple times and check the phc2sys process openshift_ptp_clock_state between iterations
      
          

      Actual results:

          openshift_ptp_clock_state{iface="CLOCK_REALTIME",node="api.kni-qe-74.lab.eng.rdu2.redhat.com",process="phc2sys"} 0

      Expected results:

          openshift_ptp_clock_state{iface="CLOCK_REALTIME",node="api.kni-qe-74.lab.eng.rdu2.redhat.com",process="phc2sys"} 1

      Additional info:

      Attaching must-gather.
      
      test: https://github.com/openshift-kni/eco-gotests/blob/main/tests/system-tests/ran-du/tests/launch-workload-multiple-iter-loadavg.go

              aputtur@redhat.com Aneesh Puttur
              mcornea@redhat.com Marius Cornea
              Bonnie Block Bonnie Block
              Votes:
              0 Vote for this issue
              Watchers:
              7 Start watching this issue

                Created:
                Updated:
                Resolved: