Uploaded image for project: 'OpenShift API for Data Protection'
  1. OpenShift API for Data Protection
  2. OADP-3767

Cassandra application is going in crashloopbackoff after restore

XMLWordPrintable

    • 3
    • False
    • Hide

      None

      Show
      None
    • False
    • ToDo
    • Known Issue
    • 2
    • 0
    • 0.000
    • Very Likely
    • 0
    • None
    • Unset
    • Unknown
    • Yes

      Description of problem:

      Cassandra tests started failing on ROSA 4.14 cluster. The issue has not been seen in ROSA 4.15 cluster. The pods are going back in crashloopbackoff status after restore which causing the tests to fail. 

      Attached cassandra pod logs below.

      $ oc logs cassandra-2 -n ocp-cassandra --previous
      OpenJDK 64-Bit Server VM warning: MaxNewSize (524288k) is equal to or greater than the entire heap (524288k).  A new max generation size of 524224k will be used.
      CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.deserializeLargeSubset (Lorg/apache/cassandra/io/util/DataInputPlus;Lorg/apache/cassandra/db/Columns;I)Lorg/apache/cassandra/db/Columns;
      CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.serializeLargeSubset (Ljava/util/Collection;ILorg/apache/cassandra/db/Columns;ILorg/apache/cassandra/io/util/DataOutputPlus;)V
      CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.serializeLargeSubsetSize (Ljava/util/Collection;ILorg/apache/cassandra/db/Columns;I)I
      CompilerOracle: dontinline org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.advanceAllocatingFrom (Lorg/apache/cassandra/db/commitlog/CommitLogSegment;)V
      CompilerOracle: dontinline org/apache/cassandra/db/transform/BaseIterator.tryGetMoreContents ()Z
      CompilerOracle: dontinline org/apache/cassandra/db/transform/StoppingTransformation.stop ()V
      CompilerOracle: dontinline org/apache/cassandra/db/transform/StoppingTransformation.stopInPartition ()V
      CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.doFlush (I)V
      CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.writeExcessSlow ()V
      CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.writeSlow (JI)V
      CompilerOracle: dontinline org/apache/cassandra/io/util/RebufferingInputStream.readPrimitiveSlowly (I)J
      CompilerOracle: inline org/apache/cassandra/db/rows/UnfilteredSerializer.serializeRowBody (Lorg/apache/cassandra/db/rows/Row;ILorg/apache/cassandra/db/SerializationHeader;Lorg/apache/cassandra/io/util/DataOutputPlus;)V
      CompilerOracle: inline org/apache/cassandra/io/util/Memory.checkBounds (JJ)V
      CompilerOracle: inline org/apache/cassandra/io/util/SafeMemory.checkBounds (JJ)V
      CompilerOracle: inline org/apache/cassandra/utils/AsymmetricOrdering.selectBoundary (Lorg/apache/cassandra/utils/AsymmetricOrdering/Op;II)I
      CompilerOracle: inline org/apache/cassandra/utils/AsymmetricOrdering.strictnessOfLessThan (Lorg/apache/cassandra/utils/AsymmetricOrdering/Op;)I
      CompilerOracle: inline org/apache/cassandra/utils/BloomFilter.indexes (Lorg/apache/cassandra/utils/IFilter/FilterKey;)[J
      CompilerOracle: inline org/apache/cassandra/utils/BloomFilter.setIndexes (JJIJ[J)V
      CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compare (Ljava/nio/ByteBuffer;[B)I
      CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compare ([BLjava/nio/ByteBuffer;)I
      CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compareUnsigned (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)I
      CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/lang/Object;JILjava/lang/Object;JI)I
      CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/lang/Object;JILjava/nio/ByteBuffer;)I
      CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)I
      CompilerOracle: inline org/apache/cassandra/utils/vint/VIntCoding.encodeVInt (JI)[B
      INFO  [main] 2024-03-27 11:36:21,459 YamlConfigurationLoader.java:93 - Configuration location: file:/etc/cassandra/cassandra.yaml
      INFO  [main] 2024-03-27 11:36:22,751 Config.java:555 - Node configuration:[allocate_tokens_for_keyspace=null; allow_extra_insecure_udfs=false; allow_insecure_udfs=false; authenticator=AllowAllAuthenticator; authorizer=AllowAllAuthorizer; auto_bootstrap=true; auto_snapshot=true; back_pressure_enabled=false; back_pressure_strategy=org.apache.cassandra.net.RateBasedBackPressure{high_ratio=0.9, factor=5, flow=FAST}; batch_size_fail_threshold_in_kb=50; batch_size_warn_threshold_in_kb=5; batchlog_replay_throttle_in_kb=1024; broadcast_address=10.128.2.27; broadcast_rpc_address=10.128.2.27; buffer_pool_use_heap_if_exhausted=true; cache_load_timeout_seconds=30; cas_contention_timeout_in_ms=1000; cdc_enabled=false; cdc_free_space_check_interval_ms=250; cdc_raw_directory=null; cdc_total_space_in_mb=0; check_for_duplicate_rows_during_compaction=true; check_for_duplicate_rows_during_reads=true; client_encryption_options=<REDACTED>; cluster_name=Cassandra; column_index_cache_size_in_kb=2; column_index_size_in_kb=64; commit_failure_policy=stop; commitlog_compression=null; commitlog_directory=null; commitlog_max_compression_buffers_in_pool=3; commitlog_periodic_queue_size=-1; commitlog_segment_size_in_mb=32; commitlog_sync=periodic; commitlog_sync_batch_window_in_ms=NaN; commitlog_sync_period_in_ms=10000; commitlog_total_space_in_mb=null; compaction_large_partition_warning_threshold_mb=100; compaction_throughput_mb_per_sec=16; concurrent_compactors=null; concurrent_counter_writes=32; concurrent_materialized_view_writes=32; concurrent_reads=32; concurrent_replicates=null; concurrent_writes=32; counter_cache_keys_to_save=2147483647; counter_cache_save_period=7200; counter_cache_size_in_mb=null; counter_write_request_timeout_in_ms=5000; credentials_cache_max_entries=1000; credentials_update_interval_in_ms=-1; credentials_validity_in_ms=2000; cross_node_timeout=false; data_file_directories=[Ljava.lang.String;@7219ec67; disk_access_mode=auto; disk_failure_policy=stop; disk_optimization_estimate_percentile=0.95; disk_optimization_page_cross_chance=0.1; disk_optimization_strategy=ssd; dynamic_snitch=true; dynamic_snitch_badness_threshold=0.1; dynamic_snitch_reset_interval_in_ms=600000; dynamic_snitch_update_interval_in_ms=100; enable_drop_compact_storage=false; enable_materialized_views=true; enable_sasi_indexes=true; enable_scripted_user_defined_functions=false; enable_user_defined_functions=false; enable_user_defined_functions_threads=true; encryption_options=<REDACTED>; endpoint_snitch=GossipingPropertyFileSnitch; file_cache_round_up=null; file_cache_size_in_mb=null; force_new_prepared_statement_behaviour=false; gc_log_threshold_in_ms=200; gc_warn_threshold_in_ms=1000; hinted_handoff_disabled_datacenters=[]; hinted_handoff_enabled=true; hinted_handoff_throttle_in_kb=1024; hints_compression=null; hints_directory=null; hints_flush_period_in_ms=10000; incremental_backups=false; index_interval=null; index_summary_capacity_in_mb=null; index_summary_resize_interval_in_minutes=60; initial_token=null; inter_dc_stream_throughput_outbound_megabits_per_sec=200; inter_dc_tcp_nodelay=false; internode_authenticator=null; internode_compression=dc; internode_recv_buff_size_in_bytes=0; internode_send_buff_size_in_bytes=0; key_cache_keys_to_save=2147483647; key_cache_save_period=14400; key_cache_size_in_mb=null; listen_address=10.128.2.27; listen_interface=null; listen_interface_prefer_ipv6=false; listen_on_broadcast_address=false; max_hint_window_in_ms=10800000; max_hints_delivery_threads=2; max_hints_file_size_in_mb=128; max_mutation_size_in_kb=null; max_streaming_retries=3; max_value_size_in_mb=256; memtable_allocation_type=heap_buffers; memtable_cleanup_threshold=null; memtable_flush_writers=0; memtable_heap_space_in_mb=null; memtable_offheap_space_in_mb=null; min_free_space_per_drive_in_mb=50; native_transport_flush_in_batches_legacy=true; native_transport_max_concurrent_connections=-1; native_transport_max_concurrent_connections_per_ip=-1; native_transport_max_concurrent_requests_in_bytes=-1; native_transport_max_concurrent_requests_in_bytes_per_ip=-1; native_transport_max_frame_size_in_mb=256; native_transport_max_negotiable_protocol_version=-2147483648; native_transport_max_threads=128; native_transport_port=9042; native_transport_port_ssl=null; num_tokens=256; otc_backlog_expiration_interval_ms=200; otc_coalescing_enough_coalesced_messages=8; otc_coalescing_strategy=DISABLED; otc_coalescing_window_us=200; partitioner=org.apache.cassandra.dht.Murmur3Partitioner; permissions_cache_max_entries=1000; permissions_update_interval_in_ms=-1; permissions_validity_in_ms=2000; phi_convict_threshold=8.0; prepared_statements_cache_size_mb=null; range_request_timeout_in_ms=10000; read_request_timeout_in_ms=5000; repair_session_max_tree_depth=18; request_scheduler=org.apache.cassandra.scheduler.NoScheduler; request_scheduler_id=null; request_scheduler_options=null; request_timeout_in_ms=10000; role_manager=CassandraRoleManager; roles_cache_max_entries=1000; roles_update_interval_in_ms=-1; roles_validity_in_ms=2000; row_cache_class_name=org.apache.cassandra.cache.OHCProvider; row_cache_keys_to_save=2147483647; row_cache_save_period=0; row_cache_size_in_mb=0; rpc_address=0.0.0.0; rpc_interface=null; rpc_interface_prefer_ipv6=false; rpc_keepalive=true; rpc_listen_backlog=50; rpc_max_threads=2147483647; rpc_min_threads=16; rpc_port=9160; rpc_recv_buff_size_in_bytes=null; rpc_send_buff_size_in_bytes=null; rpc_server_type=sync; saved_caches_directory=null; seed_provider=org.apache.cassandra.locator.SimpleSeedProvider{seeds=cassandra-0.cassandra.ocp-cassandra.svc.cluster.local}; server_encryption_options=<REDACTED>; slow_query_log_timeout_in_ms=500; snapshot_before_compaction=false; snapshot_on_duplicate_row_detection=false; ssl_storage_port=7001; sstable_preemptive_open_interval_in_mb=50; start_native_transport=true; start_rpc=false; storage_port=7000; stream_throughput_outbound_megabits_per_sec=200; streaming_keep_alive_period_in_secs=300; streaming_socket_timeout_in_ms=86400000; thrift_framed_transport_size_in_mb=15; thrift_max_message_length_in_mb=16; thrift_prepared_statements_cache_size_mb=null; tombstone_failure_threshold=100000; tombstone_warn_threshold=1000; tracetype_query_ttl=86400; tracetype_repair_ttl=604800; transparent_data_encryption_options=org.apache.cassandra.config.TransparentDataEncryptionOptions@45018215; trickle_fsync=false; trickle_fsync_interval_in_kb=10240; truncate_request_timeout_in_ms=60000; unlogged_batch_across_partitions_warn_threshold=10; user_defined_function_fail_timeout=1500; user_defined_function_warn_timeout=500; user_function_timeout_policy=die; windows_timer_interval=1; write_request_timeout_in_ms=2000]
      INFO  [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:381 - DiskAccessMode 'auto' determined to be mmap, indexAccessMode is mmap
      INFO  [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:439 - Global memtable on-heap threshold is enabled at 115MB
      INFO  [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:443 - Global memtable off-heap threshold is enabled at 115MB
      WARN  [main] 2024-03-27 11:36:22,756 DatabaseDescriptor.java:503 - Small commitlog volume detected at /opt/cassandra/data/commitlog; setting commitlog_total_space_in_mb to 243.  You can override this in cassandra.yaml
      WARN  [main] 2024-03-27 11:36:22,757 DatabaseDescriptor.java:530 - Small cdc volume detected at /opt/cassandra/data/cdc_raw; setting cdc_total_space_in_mb to 121.  You can override this in cassandra.yaml
      WARN  [main] 2024-03-27 11:36:23,058 DatabaseDescriptor.java:579 - Only 971.797MiB free across all data volumes. Consider adding more capacity to your cluster or removing obsolete snapshots
      INFO  [main] 2024-03-27 11:36:23,076 RateBasedBackPressure.java:123 - Initialized back-pressure with high ratio: 0.9, factor: 5, flow: FAST, window size: 2000.
      INFO  [main] 2024-03-27 11:36:23,076 DatabaseDescriptor.java:781 - Back-pressure is disabled with strategy org.apache.cassandra.net.RateBasedBackPressure{high_ratio=0.9, factor=5, flow=FAST}.
      INFO  [main] 2024-03-27 11:36:23,440 GossipingPropertyFileSnitch.java:64 - Loaded cassandra-topology.properties for compatibility
      INFO  [ScheduledTasks:1] 2024-03-27 11:36:28,453 TokenMetadata.java:526 - Updating topology for all endpoints that have changed
      WARN  [main] 2024-03-27 11:36:43,561 SimpleSeedProvider.java:60 - Seed provider couldn't lookup host cassandra-0.cassandra.ocp-cassandra.svc.cluster.local
      Exception (org.apache.cassandra.exceptions.ConfigurationException) encountered during startup: The seed provider lists no seeds.
      The seed provider lists no seeds.
      ERROR [main] 2024-03-27 11:36:43,563 CassandraDaemon.java:803 - Exception encountered during startup: The seed provider lists no seeds.// code placeholder
      

       

       

      Version-Release number of selected component (if applicable):
      ROSA 4.14 cluster
      OADP 1.3.1-52 stage build (I have also seen the similar issue with 1.3.1-49 build)

       

      How reproducible:
      Always (I tried it couple of times)

       

      Steps to Reproduce:
      1. Deploy a ROSA 4.14 cluster. Install OADP operator with standardized CCO configuration
      2. Deploy the cassandra application. 

      $ appm deploy ocp-cassandra -ntest2

      3. Execute CSI backup

      4. Delete app namespace and execute restore.
      5. Verify the application after restore

      $ appm validate ocp-cassandra -n test2

       

      Actual results:
      Pod is going in a crashloopbackoff status

      $ oc get pod -n test2
      NAME          READY   STATUS             RESTARTS      AGE
      cassandra-0   1/1     Running            0             9m18s
      cassandra-1   0/1     CrashLoopBackOff   6 (39s ago)   9m18s
      cassandra-2   0/1     CrashLoopBackOff   6 (39s ago)   9m18s

      Expected results:

      There should be no crashloopbackoff after restore. 

      Additional info:

      $ velero describe backup test-backup2 -n openshift-adp --details
      Name:         test-backup2
      Namespace:    openshift-adp
      Labels:       velero.io/storage-location=ts-dpa-1
      Annotations:  velero.io/resource-timeout=10m0s
                    velero.io/source-cluster-k8s-gitversion=v1.27.10+c79e5e2
                    velero.io/source-cluster-k8s-major-version=1
                    velero.io/source-cluster-k8s-minor-version=27
      
      Phase:  Completed
      
      
      Namespaces:
        Included:  test2
        Excluded:  <none>
      
      Resources:
        Included:        *
        Excluded:        <none>
        Cluster-scoped:  auto
      
      Label selector:  <none>
      
      Or label selector:  <none>
      
      Storage Location:  ts-dpa-1
      
      Velero-Native Snapshot PVs:  auto
      Snapshot Move Data:          false
      Data Mover:                  velero
      
      TTL:  720h0m0s
      
      CSISnapshotTimeout:    10m0s
      ItemOperationTimeout:  4h0m0s
      
      Hooks:  <none>
      
      Backup Format Version:  1.1.0
      
      Started:    2024-03-27 18:17:45 +0530 IST
      Completed:  2024-03-27 18:18:36 +0530 IST
      
      Expiration:  2024-04-26 18:17:45 +0530 IST
      
      Total items to be backed up:  96
      Items backed up:              96
      
      Backup Item Operations:
        Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-0-4zmqt:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshot-backupper
          Operation ID:               test2/velero-cassandra-data-cassandra-0-4zmqt/2024-03-27T12:47:56Z
          Items to Update:
                    volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-0-4zmqt
          Phase:    Completed
          Created:  2024-03-27 18:17:56 +0530 IST
          Started:  2024-03-27 18:17:56 +0530 IST
        Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-9feaa340-a009-4290-808d-05b030a91072:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshotcontent-backupper
          Operation ID:               snapcontent-9feaa340-a009-4290-808d-05b030a91072/2024-03-27T12:47:56Z
          Items to Update:
                    volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-9feaa340-a009-4290-808d-05b030a91072
          Phase:    Completed
          Created:  2024-03-27 18:17:56 +0530 IST
          Started:  2024-03-27 18:17:56 +0530 IST
        Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-1-qjx6k:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshot-backupper
          Operation ID:               test2/velero-cassandra-data-cassandra-1-qjx6k/2024-03-27T12:48:01Z
          Items to Update:
                    volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-1-qjx6k
          Phase:    Completed
          Created:  2024-03-27 18:18:01 +0530 IST
          Started:  2024-03-27 18:18:01 +0530 IST
        Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshotcontent-backupper
          Operation ID:               snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401/2024-03-27T12:48:01Z
          Items to Update:
                    volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401
          Phase:    Completed
          Created:  2024-03-27 18:18:01 +0530 IST
          Started:  2024-03-27 18:18:01 +0530 IST
        Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-2-k5rbn:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshot-backupper
          Operation ID:               test2/velero-cassandra-data-cassandra-2-k5rbn/2024-03-27T12:48:06Z
          Items to Update:
                    volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-2-k5rbn
          Phase:    Completed
          Created:  2024-03-27 18:18:06 +0530 IST
          Started:  2024-03-27 18:18:06 +0530 IST
        Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8:
          Backup Item Action Plugin:  velero.io/csi-volumesnapshotcontent-backupper
          Operation ID:               snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8/2024-03-27T12:48:06Z
          Items to Update:
                    volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8
          Phase:    Completed
          Created:  2024-03-27 18:18:06 +0530 IST
          Started:  2024-03-27 18:18:06 +0530 IST
      Resource List:
        apiextensions.k8s.io/v1/CustomResourceDefinition:
          - clusterserviceversions.operators.coreos.com
        apps/v1/ControllerRevision:
          - test2/cassandra-846b97fbc
        apps/v1/StatefulSet:
          - test2/cassandra
        authorization.openshift.io/v1/RoleBinding:
          - test2/admin
          - test2/admin-dedicated-admins
          - test2/admin-system:serviceaccounts:dedicated-admin
          - test2/alert-routing-edit-dedicated-admins
          - test2/dedicated-admins-project-dedicated-admins
          - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin
          - test2/system:deployers
          - test2/system:image-builders
          - test2/system:image-pullers
          - test2/system:openshift:scc:privileged
        discovery.k8s.io/v1/EndpointSlice:
          - test2/cassandra-cmhg5
        operators.coreos.com/v1alpha1/ClusterServiceVersion:
          - test2/observability-operator.v0.0.28
          - test2/route-monitor-operator.v0.1.632-df5f264
        rbac.authorization.k8s.io/v1/RoleBinding:
          - test2/admin
          - test2/admin-dedicated-admins
          - test2/admin-system:serviceaccounts:dedicated-admin
          - test2/alert-routing-edit-dedicated-admins
          - test2/dedicated-admins-project-dedicated-admins
          - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin
          - test2/system:deployers
          - test2/system:image-builders
          - test2/system:image-pullers
          - test2/system:openshift:scc:privileged
        snapshot.storage.k8s.io/v1/VolumeSnapshot:
          - test2/velero-cassandra-data-cassandra-0-4zmqt
          - test2/velero-cassandra-data-cassandra-1-qjx6k
          - test2/velero-cassandra-data-cassandra-2-k5rbn
        snapshot.storage.k8s.io/v1/VolumeSnapshotClass:
          - csi-aws-vsc
        snapshot.storage.k8s.io/v1/VolumeSnapshotContent:
          - snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401
          - snapcontent-9feaa340-a009-4290-808d-05b030a91072
          - snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8
        v1/ConfigMap:
          - test2/kube-root-ca.crt
          - test2/openshift-service-ca.crt
        v1/Endpoints:
          - test2/cassandra
        v1/Event:
          - test2/cassandra-0.17c09f55d9832657
          - test2/cassandra-0.17c09f5664c1dc76
          - test2/cassandra-0.17c09f570fa73c1f
          - test2/cassandra-0.17c09f57114ffde3
          - test2/cassandra-0.17c09f61aa5618e8
          - test2/cassandra-0.17c09f61b08f8ab7
          - test2/cassandra-0.17c09f61b185a061
          - test2/cassandra-1.17c09f62c544f293
          - test2/cassandra-1.17c09f634e29dc91
          - test2/cassandra-1.17c09f63e9529083
          - test2/cassandra-1.17c09f63eaff7dc8
          - test2/cassandra-1.17c09f63ff29e827
          - test2/cassandra-1.17c09f64055c22d2
          - test2/cassandra-1.17c09f640642aa10
          - test2/cassandra-2.17c09f65270f8a38
          - test2/cassandra-2.17c09f65b2836c64
          - test2/cassandra-2.17c09f67340d0ac7
          - test2/cassandra-2.17c09f6735bd4461
          - test2/cassandra-2.17c09f6870c443de
          - test2/cassandra-2.17c09f6877ce44b3
          - test2/cassandra-2.17c09f6878de3a7f
          - test2/cassandra-2.17c09f6db84f98f5
          - test2/cassandra-data-cassandra-0.17c09f54e9547335
          - test2/cassandra-data-cassandra-0.17c09f54eb5d0f9a
          - test2/cassandra-data-cassandra-0.17c09f54eb5eaeb7
          - test2/cassandra-data-cassandra-0.17c09f55ba350fe2
          - test2/cassandra-data-cassandra-1.17c09f61d5134475
          - test2/cassandra-data-cassandra-1.17c09f61d6fdc837
          - test2/cassandra-data-cassandra-1.17c09f61d700ca7f
          - test2/cassandra-data-cassandra-1.17c09f62a506095b
          - test2/cassandra-data-cassandra-2.17c09f6437106842
          - test2/cassandra-data-cassandra-2.17c09f64389639c7
          - test2/cassandra-data-cassandra-2.17c09f6438967fee
          - test2/cassandra-data-cassandra-2.17c09f6500affef6
          - test2/cassandra.17c09f54e954b7b0
          - test2/cassandra.17c09f54ea177adc
          - test2/cassandra.17c09f61d5121f05
          - test2/cassandra.17c09f61d5d4d5d3
          - test2/cassandra.17c09f64370f737f
          - test2/cassandra.17c09f6437b10b0d
        v1/Namespace:
          - test2
        v1/PersistentVolume:
          - pvc-2d084ccd-24c3-4ae5-863f-0f05a58801c4
          - pvc-51740cf1-f13a-4c3e-8777-d67ee2a8ecaa
          - pvc-60f779f3-cadc-4ea4-873c-1278946a5ece
        v1/PersistentVolumeClaim:
          - test2/cassandra-data-cassandra-0
          - test2/cassandra-data-cassandra-1
          - test2/cassandra-data-cassandra-2
        v1/Pod:
          - test2/cassandra-0
          - test2/cassandra-1
          - test2/cassandra-2
        v1/Secret:
          - test2/builder-dockercfg-7vl2n
          - test2/builder-token-n5dsd
          - test2/default-dockercfg-cjs2m
          - test2/default-token-vpt8x
          - test2/deployer-dockercfg-hwjtq
          - test2/deployer-token-26wdw
        v1/Service:
          - test2/cassandra
        v1/ServiceAccount:
          - test2/builder
          - test2/default
          - test2/deployer
      
      Velero-Native Snapshots: <none included> 
      $ velero describe restore test-restore2 -n openshift-adp --details
      Name:         test-restore2
      Namespace:    openshift-adp
      Labels:       <none>
      Annotations:  <none>
      
      Phase:                       Completed
      Total items to be restored:  56
      Items restored:              56
      
      Started:    2024-03-27 18:29:41 +0530 IST
      Completed:  2024-03-27 18:30:01 +0530 IST
      
      Warnings:
        Velero:     <none>
        Cluster:  could not restore, CustomResourceDefinition "clusterserviceversions.operators.coreos.com" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, VolumeSnapshotContent "snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, VolumeSnapshotContent "snapcontent-9feaa340-a009-4290-808d-05b030a91072" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, VolumeSnapshotContent "snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8" already exists. Warning: the in-cluster version is different than the backed-up version
        Namespaces:
          test2:  could not restore, RoleBinding "dedicated-admins-project-dedicated-admins" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, RoleBinding "system:image-pullers" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, ConfigMap "kube-root-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version
                  could not restore, ConfigMap "openshift-service-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin-dedicated-admins: rolebindings.authorization.openshift.io "admin-dedicated-admins" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin-system:serviceaccounts:dedicated-admin: rolebindings.authorization.openshift.io "admin-system:serviceaccounts:dedicated-admin" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin: rolebindings.authorization.openshift.io "admin" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/alert-routing-edit-dedicated-admins: rolebindings.authorization.openshift.io "alert-routing-edit-dedicated-admins" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/dedicated-admins-project-dedicated-admins: rolebindings.authorization.openshift.io "dedicated-admins-project-dedicated-admins" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin: rolebindings.authorization.openshift.io "dedicated-admins-project-system:serviceaccounts:dedicated-admin" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:deployers: rolebindings.authorization.openshift.io "system:deployers" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:image-builders: rolebindings.authorization.openshift.io "system:image-builders" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:image-pullers: rolebindings.authorization.openshift.io "system:image-pullers" not found
                  error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:openshift:scc:privileged: rolebindings.authorization.openshift.io "system:openshift:scc:privileged" not found
      
      Backup:  test-backup2
      
      Namespaces:
        Included:  all namespaces found in the backup
        Excluded:  <none>
      
      Resources:
        Included:        *
        Excluded:        nodes, events, events.events.k8s.io, backups.velero.io, restores.velero.io, resticrepositories.velero.io, csinodes.storage.k8s.io, volumeattachments.storage.k8s.io, backuprepositories.velero.io
        Cluster-scoped:  auto
      
      Namespace mappings:  <none>
      
      Label selector:  <none>
      
      Or label selector:  <none>
      
      Restore PVs:  auto
      
      Existing Resource Policy:   <none>
      ItemOperationTimeout:       4h0m0s
      
      Preserve Service NodePorts:  auto
      
      Resource List:
        apiextensions.k8s.io/v1/CustomResourceDefinition:
          - clusterserviceversions.operators.coreos.com(failed)
        apps/v1/ControllerRevision:
          - test2/cassandra-846b97fbc(created)
        apps/v1/StatefulSet:
          - test2/cassandra(created)
        authorization.openshift.io/v1/RoleBinding:
          - test2/admin(failed)
          - test2/admin-dedicated-admins(failed)
          - test2/admin-system:serviceaccounts:dedicated-admin(failed)
          - test2/alert-routing-edit-dedicated-admins(failed)
          - test2/dedicated-admins-project-dedicated-admins(failed)
          - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin(failed)
          - test2/system:deployers(failed)
          - test2/system:image-builders(failed)
          - test2/system:image-pullers(failed)
          - test2/system:openshift:scc:privileged(failed)
        discovery.k8s.io/v1/EndpointSlice:
          - test2/cassandra-cmhg5(created)
        operators.coreos.com/v1alpha1/ClusterServiceVersion:
          - test2/observability-operator.v0.0.28(created)
          - test2/route-monitor-operator.v0.1.632-df5f264(created)
        rbac.authorization.k8s.io/v1/RoleBinding:
          - test2/admin(created)
          - test2/admin-dedicated-admins(created)
          - test2/admin-system:serviceaccounts:dedicated-admin(created)
          - test2/alert-routing-edit-dedicated-admins(created)
          - test2/dedicated-admins-project-dedicated-admins(failed)
          - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin(created)
          - test2/system:deployers(created)
          - test2/system:image-builders(created)
          - test2/system:image-pullers(failed)
          - test2/system:openshift:scc:privileged(created)
        snapshot.storage.k8s.io/v1/VolumeSnapshot:
          - test2/velero-cassandra-data-cassandra-0-4zmqt(created)
          - test2/velero-cassandra-data-cassandra-1-qjx6k(created)
          - test2/velero-cassandra-data-cassandra-2-k5rbn(created)
        snapshot.storage.k8s.io/v1/VolumeSnapshotClass:
          - csi-aws-vsc(skipped)
        snapshot.storage.k8s.io/v1/VolumeSnapshotContent:
          - snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401(failed)
          - snapcontent-9feaa340-a009-4290-808d-05b030a91072(failed)
          - snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8(failed)
        v1/ConfigMap:
          - test2/kube-root-ca.crt(failed)
          - test2/openshift-service-ca.crt(failed)
        v1/Endpoints:
          - test2/cassandra(created)
        v1/Namespace:
          - test2(created)
        v1/PersistentVolume:
          - pvc-2d084ccd-24c3-4ae5-863f-0f05a58801c4(skipped)
          - pvc-51740cf1-f13a-4c3e-8777-d67ee2a8ecaa(skipped)
          - pvc-60f779f3-cadc-4ea4-873c-1278946a5ece(skipped)
        v1/PersistentVolumeClaim:
          - test2/cassandra-data-cassandra-0(created)
          - test2/cassandra-data-cassandra-1(created)
          - test2/cassandra-data-cassandra-2(created)
        v1/Pod:
          - test2/cassandra-0(created)
          - test2/cassandra-1(created)
          - test2/cassandra-2(created)
        v1/Secret:
          - test2/builder-dockercfg-7vl2n(created)
          - test2/builder-token-n5dsd(skipped)
          - test2/default-dockercfg-cjs2m(created)
          - test2/default-token-vpt8x(skipped)
          - test2/deployer-dockercfg-hwjtq(created)
          - test2/deployer-token-26wdw(skipped)
        v1/Service:
          - test2/cassandra(created)
        v1/ServiceAccount:
          - test2/builder(skipped)
          - test2/default(skipped)
          - test2/deployer(skipped)

      StatefulSet yaml

      $ oc get statefulset -n ocp-cassandra cassandra -o yaml
      apiVersion: apps/v1
      kind: StatefulSet
      metadata:
        annotations:
          openshift.io/backup-registry-hostname: image-registry.openshift-image-registry.svc:5000
          openshift.io/backup-server-version: "1.27"
          openshift.io/restore-registry-hostname: image-registry.openshift-image-registry.svc:5000
          openshift.io/restore-server-version: "1.27"
        creationTimestamp: "2024-03-27T06:24:09Z"
        generation: 1
        labels:
          app: cassandra
          velero.io/backup-name: test-backup
          velero.io/restore-name: test-restore
        name: cassandra
        namespace: ocp-cassandra
        resourceVersion: "352983"
        uid: 8dd5868d-f67c-413e-8426-0c02c31d10a9
      spec:
        persistentVolumeClaimRetentionPolicy:
          whenDeleted: Retain
          whenScaled: Retain
        podManagementPolicy: OrderedReady
        replicas: 3
        revisionHistoryLimit: 10
        selector:
          matchLabels:
            app: cassandra
        serviceName: cassandra
        template:
          metadata:
            creationTimestamp: null
            labels:
              app: cassandra
          spec:
            containers:
            - env:
              - name: MAX_HEAP_SIZE
                value: 512M
              - name: HEAP_NEWSIZE
                value: 512M
              - name: CASSANDRA_SEEDS
                value: cassandra-0.cassandra.ocp-cassandra.svc.cluster.local
              - name: CASSANDRA_CLUSTER_NAME
                value: Cassandra
              - name: CASSANDRA_DC
                value: DC1
              - name: CASSANDRA_RACK
                value: Rack1
              - name: CASSANDRA_AUTO_BOOTSTRAP
                value: "false"
              - name: CASSANDRA_ENDPOINT_SNITCH
                value: GossipingPropertyFileSnitch
              - name: POD_IP
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: status.podIP
              image: quay.io/migqe/cassandra:multiarch
              imagePullPolicy: Always
              lifecycle:
                preStop:
                  exec:
                    command:
                    - /bin/sh
                    - -c
                    - nodetool drain
              name: cassandra
              ports:
              - containerPort: 7000
                name: intra-node
                protocol: TCP
              - containerPort: 7001
                name: tls-intra-node
                protocol: TCP
              - containerPort: 7199
                name: jmx
                protocol: TCP
              - containerPort: 9042
                name: cql
                protocol: TCP
              resources:
                limits:
                  cpu: 500m
                  memory: 1Gi
                requests:
                  cpu: 500m
                  memory: 1Gi
              securityContext:
                privileged: true
              terminationMessagePath: /dev/termination-log
              terminationMessagePolicy: File
              volumeMounts:
              - mountPath: /var/lib/cassandra
                name: cassandra-data
            dnsPolicy: ClusterFirst
            restartPolicy: Always
            schedulerName: default-scheduler
            securityContext: {}
            terminationGracePeriodSeconds: 1800
        updateStrategy:
          rollingUpdate:
            partition: 0
          type: RollingUpdate
        volumeClaimTemplates:
        - apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            creationTimestamp: null
            name: cassandra-data
            namespace: ocp-cassandra
          spec:
            accessModes:
            - ReadWriteOnce
            resources:
              requests:
                storage: 1Gi
            volumeMode: Filesystem
          status:
            phase: Pending
      status:
        availableReplicas: 1
        collisionCount: 0
        currentReplicas: 3
        currentRevision: cassandra-76bd54848b
        observedGeneration: 1
        readyReplicas: 1
        replicas: 3
        updateRevision: cassandra-76bd54848b
        updatedReplicas: 3

              abhide@redhat.com Apurva Bhide
              rhn-support-prajoshi Prasad Joshi
              Prasad Joshi Prasad Joshi
              Apurva Bhide Apurva Bhide
              Votes:
              0 Vote for this issue
              Watchers:
              7 Start watching this issue

                Created:
                Updated:
                Resolved: