-
Bug
-
Resolution: Unresolved
-
Major
-
OADP 1.3.1
-
3
-
False
-
-
False
-
ToDo
-
Known Issue
-
-
-
2
-
0
-
0.0
-
Very Likely
-
0
-
None
-
Unset
-
Unknown
-
Yes
Description of problem:
Cassandra tests started failing on ROSA 4.14 cluster. The issue has not been seen in ROSA 4.15 cluster. The pods are going back in crashloopbackoff status after restore which causing the tests to fail.
Attached cassandra pod logs below.
$ oc logs cassandra-2 -n ocp-cassandra --previous OpenJDK 64-Bit Server VM warning: MaxNewSize (524288k) is equal to or greater than the entire heap (524288k). A new max generation size of 524224k will be used. CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.deserializeLargeSubset (Lorg/apache/cassandra/io/util/DataInputPlus;Lorg/apache/cassandra/db/Columns;I)Lorg/apache/cassandra/db/Columns; CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.serializeLargeSubset (Ljava/util/Collection;ILorg/apache/cassandra/db/Columns;ILorg/apache/cassandra/io/util/DataOutputPlus;)V CompilerOracle: dontinline org/apache/cassandra/db/Columns$Serializer.serializeLargeSubsetSize (Ljava/util/Collection;ILorg/apache/cassandra/db/Columns;I)I CompilerOracle: dontinline org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.advanceAllocatingFrom (Lorg/apache/cassandra/db/commitlog/CommitLogSegment;)V CompilerOracle: dontinline org/apache/cassandra/db/transform/BaseIterator.tryGetMoreContents ()Z CompilerOracle: dontinline org/apache/cassandra/db/transform/StoppingTransformation.stop ()V CompilerOracle: dontinline org/apache/cassandra/db/transform/StoppingTransformation.stopInPartition ()V CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.doFlush (I)V CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.writeExcessSlow ()V CompilerOracle: dontinline org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.writeSlow (JI)V CompilerOracle: dontinline org/apache/cassandra/io/util/RebufferingInputStream.readPrimitiveSlowly (I)J CompilerOracle: inline org/apache/cassandra/db/rows/UnfilteredSerializer.serializeRowBody (Lorg/apache/cassandra/db/rows/Row;ILorg/apache/cassandra/db/SerializationHeader;Lorg/apache/cassandra/io/util/DataOutputPlus;)V CompilerOracle: inline org/apache/cassandra/io/util/Memory.checkBounds (JJ)V CompilerOracle: inline org/apache/cassandra/io/util/SafeMemory.checkBounds (JJ)V CompilerOracle: inline org/apache/cassandra/utils/AsymmetricOrdering.selectBoundary (Lorg/apache/cassandra/utils/AsymmetricOrdering/Op;II)I CompilerOracle: inline org/apache/cassandra/utils/AsymmetricOrdering.strictnessOfLessThan (Lorg/apache/cassandra/utils/AsymmetricOrdering/Op;)I CompilerOracle: inline org/apache/cassandra/utils/BloomFilter.indexes (Lorg/apache/cassandra/utils/IFilter/FilterKey;)[J CompilerOracle: inline org/apache/cassandra/utils/BloomFilter.setIndexes (JJIJ[J)V CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compare (Ljava/nio/ByteBuffer;[B)I CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compare ([BLjava/nio/ByteBuffer;)I CompilerOracle: inline org/apache/cassandra/utils/ByteBufferUtil.compareUnsigned (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)I CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/lang/Object;JILjava/lang/Object;JI)I CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/lang/Object;JILjava/nio/ByteBuffer;)I CompilerOracle: inline org/apache/cassandra/utils/FastByteOperations$UnsafeOperations.compareTo (Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)I CompilerOracle: inline org/apache/cassandra/utils/vint/VIntCoding.encodeVInt (JI)[B INFO [main] 2024-03-27 11:36:21,459 YamlConfigurationLoader.java:93 - Configuration location: file:/etc/cassandra/cassandra.yaml INFO [main] 2024-03-27 11:36:22,751 Config.java:555 - Node configuration:[allocate_tokens_for_keyspace=null; allow_extra_insecure_udfs=false; allow_insecure_udfs=false; authenticator=AllowAllAuthenticator; authorizer=AllowAllAuthorizer; auto_bootstrap=true; auto_snapshot=true; back_pressure_enabled=false; back_pressure_strategy=org.apache.cassandra.net.RateBasedBackPressure{high_ratio=0.9, factor=5, flow=FAST}; batch_size_fail_threshold_in_kb=50; batch_size_warn_threshold_in_kb=5; batchlog_replay_throttle_in_kb=1024; broadcast_address=10.128.2.27; broadcast_rpc_address=10.128.2.27; buffer_pool_use_heap_if_exhausted=true; cache_load_timeout_seconds=30; cas_contention_timeout_in_ms=1000; cdc_enabled=false; cdc_free_space_check_interval_ms=250; cdc_raw_directory=null; cdc_total_space_in_mb=0; check_for_duplicate_rows_during_compaction=true; check_for_duplicate_rows_during_reads=true; client_encryption_options=<REDACTED>; cluster_name=Cassandra; column_index_cache_size_in_kb=2; column_index_size_in_kb=64; commit_failure_policy=stop; commitlog_compression=null; commitlog_directory=null; commitlog_max_compression_buffers_in_pool=3; commitlog_periodic_queue_size=-1; commitlog_segment_size_in_mb=32; commitlog_sync=periodic; commitlog_sync_batch_window_in_ms=NaN; commitlog_sync_period_in_ms=10000; commitlog_total_space_in_mb=null; compaction_large_partition_warning_threshold_mb=100; compaction_throughput_mb_per_sec=16; concurrent_compactors=null; concurrent_counter_writes=32; concurrent_materialized_view_writes=32; concurrent_reads=32; concurrent_replicates=null; concurrent_writes=32; counter_cache_keys_to_save=2147483647; counter_cache_save_period=7200; counter_cache_size_in_mb=null; counter_write_request_timeout_in_ms=5000; credentials_cache_max_entries=1000; credentials_update_interval_in_ms=-1; credentials_validity_in_ms=2000; cross_node_timeout=false; data_file_directories=[Ljava.lang.String;@7219ec67; disk_access_mode=auto; disk_failure_policy=stop; disk_optimization_estimate_percentile=0.95; disk_optimization_page_cross_chance=0.1; disk_optimization_strategy=ssd; dynamic_snitch=true; dynamic_snitch_badness_threshold=0.1; dynamic_snitch_reset_interval_in_ms=600000; dynamic_snitch_update_interval_in_ms=100; enable_drop_compact_storage=false; enable_materialized_views=true; enable_sasi_indexes=true; enable_scripted_user_defined_functions=false; enable_user_defined_functions=false; enable_user_defined_functions_threads=true; encryption_options=<REDACTED>; endpoint_snitch=GossipingPropertyFileSnitch; file_cache_round_up=null; file_cache_size_in_mb=null; force_new_prepared_statement_behaviour=false; gc_log_threshold_in_ms=200; gc_warn_threshold_in_ms=1000; hinted_handoff_disabled_datacenters=[]; hinted_handoff_enabled=true; hinted_handoff_throttle_in_kb=1024; hints_compression=null; hints_directory=null; hints_flush_period_in_ms=10000; incremental_backups=false; index_interval=null; index_summary_capacity_in_mb=null; index_summary_resize_interval_in_minutes=60; initial_token=null; inter_dc_stream_throughput_outbound_megabits_per_sec=200; inter_dc_tcp_nodelay=false; internode_authenticator=null; internode_compression=dc; internode_recv_buff_size_in_bytes=0; internode_send_buff_size_in_bytes=0; key_cache_keys_to_save=2147483647; key_cache_save_period=14400; key_cache_size_in_mb=null; listen_address=10.128.2.27; listen_interface=null; listen_interface_prefer_ipv6=false; listen_on_broadcast_address=false; max_hint_window_in_ms=10800000; max_hints_delivery_threads=2; max_hints_file_size_in_mb=128; max_mutation_size_in_kb=null; max_streaming_retries=3; max_value_size_in_mb=256; memtable_allocation_type=heap_buffers; memtable_cleanup_threshold=null; memtable_flush_writers=0; memtable_heap_space_in_mb=null; memtable_offheap_space_in_mb=null; min_free_space_per_drive_in_mb=50; native_transport_flush_in_batches_legacy=true; native_transport_max_concurrent_connections=-1; native_transport_max_concurrent_connections_per_ip=-1; native_transport_max_concurrent_requests_in_bytes=-1; native_transport_max_concurrent_requests_in_bytes_per_ip=-1; native_transport_max_frame_size_in_mb=256; native_transport_max_negotiable_protocol_version=-2147483648; native_transport_max_threads=128; native_transport_port=9042; native_transport_port_ssl=null; num_tokens=256; otc_backlog_expiration_interval_ms=200; otc_coalescing_enough_coalesced_messages=8; otc_coalescing_strategy=DISABLED; otc_coalescing_window_us=200; partitioner=org.apache.cassandra.dht.Murmur3Partitioner; permissions_cache_max_entries=1000; permissions_update_interval_in_ms=-1; permissions_validity_in_ms=2000; phi_convict_threshold=8.0; prepared_statements_cache_size_mb=null; range_request_timeout_in_ms=10000; read_request_timeout_in_ms=5000; repair_session_max_tree_depth=18; request_scheduler=org.apache.cassandra.scheduler.NoScheduler; request_scheduler_id=null; request_scheduler_options=null; request_timeout_in_ms=10000; role_manager=CassandraRoleManager; roles_cache_max_entries=1000; roles_update_interval_in_ms=-1; roles_validity_in_ms=2000; row_cache_class_name=org.apache.cassandra.cache.OHCProvider; row_cache_keys_to_save=2147483647; row_cache_save_period=0; row_cache_size_in_mb=0; rpc_address=0.0.0.0; rpc_interface=null; rpc_interface_prefer_ipv6=false; rpc_keepalive=true; rpc_listen_backlog=50; rpc_max_threads=2147483647; rpc_min_threads=16; rpc_port=9160; rpc_recv_buff_size_in_bytes=null; rpc_send_buff_size_in_bytes=null; rpc_server_type=sync; saved_caches_directory=null; seed_provider=org.apache.cassandra.locator.SimpleSeedProvider{seeds=cassandra-0.cassandra.ocp-cassandra.svc.cluster.local}; server_encryption_options=<REDACTED>; slow_query_log_timeout_in_ms=500; snapshot_before_compaction=false; snapshot_on_duplicate_row_detection=false; ssl_storage_port=7001; sstable_preemptive_open_interval_in_mb=50; start_native_transport=true; start_rpc=false; storage_port=7000; stream_throughput_outbound_megabits_per_sec=200; streaming_keep_alive_period_in_secs=300; streaming_socket_timeout_in_ms=86400000; thrift_framed_transport_size_in_mb=15; thrift_max_message_length_in_mb=16; thrift_prepared_statements_cache_size_mb=null; tombstone_failure_threshold=100000; tombstone_warn_threshold=1000; tracetype_query_ttl=86400; tracetype_repair_ttl=604800; transparent_data_encryption_options=org.apache.cassandra.config.TransparentDataEncryptionOptions@45018215; trickle_fsync=false; trickle_fsync_interval_in_kb=10240; truncate_request_timeout_in_ms=60000; unlogged_batch_across_partitions_warn_threshold=10; user_defined_function_fail_timeout=1500; user_defined_function_warn_timeout=500; user_function_timeout_policy=die; windows_timer_interval=1; write_request_timeout_in_ms=2000] INFO [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:381 - DiskAccessMode 'auto' determined to be mmap, indexAccessMode is mmap INFO [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:439 - Global memtable on-heap threshold is enabled at 115MB INFO [main] 2024-03-27 11:36:22,752 DatabaseDescriptor.java:443 - Global memtable off-heap threshold is enabled at 115MB WARN [main] 2024-03-27 11:36:22,756 DatabaseDescriptor.java:503 - Small commitlog volume detected at /opt/cassandra/data/commitlog; setting commitlog_total_space_in_mb to 243. You can override this in cassandra.yaml WARN [main] 2024-03-27 11:36:22,757 DatabaseDescriptor.java:530 - Small cdc volume detected at /opt/cassandra/data/cdc_raw; setting cdc_total_space_in_mb to 121. You can override this in cassandra.yaml WARN [main] 2024-03-27 11:36:23,058 DatabaseDescriptor.java:579 - Only 971.797MiB free across all data volumes. Consider adding more capacity to your cluster or removing obsolete snapshots INFO [main] 2024-03-27 11:36:23,076 RateBasedBackPressure.java:123 - Initialized back-pressure with high ratio: 0.9, factor: 5, flow: FAST, window size: 2000. INFO [main] 2024-03-27 11:36:23,076 DatabaseDescriptor.java:781 - Back-pressure is disabled with strategy org.apache.cassandra.net.RateBasedBackPressure{high_ratio=0.9, factor=5, flow=FAST}. INFO [main] 2024-03-27 11:36:23,440 GossipingPropertyFileSnitch.java:64 - Loaded cassandra-topology.properties for compatibility INFO [ScheduledTasks:1] 2024-03-27 11:36:28,453 TokenMetadata.java:526 - Updating topology for all endpoints that have changed WARN [main] 2024-03-27 11:36:43,561 SimpleSeedProvider.java:60 - Seed provider couldn't lookup host cassandra-0.cassandra.ocp-cassandra.svc.cluster.local Exception (org.apache.cassandra.exceptions.ConfigurationException) encountered during startup: The seed provider lists no seeds. The seed provider lists no seeds. ERROR [main] 2024-03-27 11:36:43,563 CassandraDaemon.java:803 - Exception encountered during startup: The seed provider lists no seeds.// code placeholder
Version-Release number of selected component (if applicable):
ROSA 4.14 cluster
OADP 1.3.1-52 stage build (I have also seen the similar issue with 1.3.1-49 build)
How reproducible:
Always (I tried it couple of times)
Steps to Reproduce:
1. Deploy a ROSA 4.14 cluster. Install OADP operator with standardized CCO configuration
2. Deploy the cassandra application.
$ appm deploy ocp-cassandra -ntest2
3. Execute CSI backup
4. Delete app namespace and execute restore.
5. Verify the application after restore
$ appm validate ocp-cassandra -n test2
Actual results:
Pod is going in a crashloopbackoff status
$ oc get pod -n test2 NAME READY STATUS RESTARTS AGE cassandra-0 1/1 Running 0 9m18s cassandra-1 0/1 CrashLoopBackOff 6 (39s ago) 9m18s cassandra-2 0/1 CrashLoopBackOff 6 (39s ago) 9m18s
Expected results:
There should be no crashloopbackoff after restore.
Additional info:
$ velero describe backup test-backup2 -n openshift-adp --details Name: test-backup2 Namespace: openshift-adp Labels: velero.io/storage-location=ts-dpa-1 Annotations: velero.io/resource-timeout=10m0s velero.io/source-cluster-k8s-gitversion=v1.27.10+c79e5e2 velero.io/source-cluster-k8s-major-version=1 velero.io/source-cluster-k8s-minor-version=27 Phase: Completed Namespaces: Included: test2 Excluded: <none> Resources: Included: * Excluded: <none> Cluster-scoped: auto Label selector: <none> Or label selector: <none> Storage Location: ts-dpa-1 Velero-Native Snapshot PVs: auto Snapshot Move Data: false Data Mover: velero TTL: 720h0m0s CSISnapshotTimeout: 10m0s ItemOperationTimeout: 4h0m0s Hooks: <none> Backup Format Version: 1.1.0 Started: 2024-03-27 18:17:45 +0530 IST Completed: 2024-03-27 18:18:36 +0530 IST Expiration: 2024-04-26 18:17:45 +0530 IST Total items to be backed up: 96 Items backed up: 96 Backup Item Operations: Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-0-4zmqt: Backup Item Action Plugin: velero.io/csi-volumesnapshot-backupper Operation ID: test2/velero-cassandra-data-cassandra-0-4zmqt/2024-03-27T12:47:56Z Items to Update: volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-0-4zmqt Phase: Completed Created: 2024-03-27 18:17:56 +0530 IST Started: 2024-03-27 18:17:56 +0530 IST Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-9feaa340-a009-4290-808d-05b030a91072: Backup Item Action Plugin: velero.io/csi-volumesnapshotcontent-backupper Operation ID: snapcontent-9feaa340-a009-4290-808d-05b030a91072/2024-03-27T12:47:56Z Items to Update: volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-9feaa340-a009-4290-808d-05b030a91072 Phase: Completed Created: 2024-03-27 18:17:56 +0530 IST Started: 2024-03-27 18:17:56 +0530 IST Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-1-qjx6k: Backup Item Action Plugin: velero.io/csi-volumesnapshot-backupper Operation ID: test2/velero-cassandra-data-cassandra-1-qjx6k/2024-03-27T12:48:01Z Items to Update: volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-1-qjx6k Phase: Completed Created: 2024-03-27 18:18:01 +0530 IST Started: 2024-03-27 18:18:01 +0530 IST Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401: Backup Item Action Plugin: velero.io/csi-volumesnapshotcontent-backupper Operation ID: snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401/2024-03-27T12:48:01Z Items to Update: volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401 Phase: Completed Created: 2024-03-27 18:18:01 +0530 IST Started: 2024-03-27 18:18:01 +0530 IST Operation for volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-2-k5rbn: Backup Item Action Plugin: velero.io/csi-volumesnapshot-backupper Operation ID: test2/velero-cassandra-data-cassandra-2-k5rbn/2024-03-27T12:48:06Z Items to Update: volumesnapshots.snapshot.storage.k8s.io test2/velero-cassandra-data-cassandra-2-k5rbn Phase: Completed Created: 2024-03-27 18:18:06 +0530 IST Started: 2024-03-27 18:18:06 +0530 IST Operation for volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8: Backup Item Action Plugin: velero.io/csi-volumesnapshotcontent-backupper Operation ID: snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8/2024-03-27T12:48:06Z Items to Update: volumesnapshotcontents.snapshot.storage.k8s.io /snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8 Phase: Completed Created: 2024-03-27 18:18:06 +0530 IST Started: 2024-03-27 18:18:06 +0530 IST Resource List: apiextensions.k8s.io/v1/CustomResourceDefinition: - clusterserviceversions.operators.coreos.com apps/v1/ControllerRevision: - test2/cassandra-846b97fbc apps/v1/StatefulSet: - test2/cassandra authorization.openshift.io/v1/RoleBinding: - test2/admin - test2/admin-dedicated-admins - test2/admin-system:serviceaccounts:dedicated-admin - test2/alert-routing-edit-dedicated-admins - test2/dedicated-admins-project-dedicated-admins - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin - test2/system:deployers - test2/system:image-builders - test2/system:image-pullers - test2/system:openshift:scc:privileged discovery.k8s.io/v1/EndpointSlice: - test2/cassandra-cmhg5 operators.coreos.com/v1alpha1/ClusterServiceVersion: - test2/observability-operator.v0.0.28 - test2/route-monitor-operator.v0.1.632-df5f264 rbac.authorization.k8s.io/v1/RoleBinding: - test2/admin - test2/admin-dedicated-admins - test2/admin-system:serviceaccounts:dedicated-admin - test2/alert-routing-edit-dedicated-admins - test2/dedicated-admins-project-dedicated-admins - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin - test2/system:deployers - test2/system:image-builders - test2/system:image-pullers - test2/system:openshift:scc:privileged snapshot.storage.k8s.io/v1/VolumeSnapshot: - test2/velero-cassandra-data-cassandra-0-4zmqt - test2/velero-cassandra-data-cassandra-1-qjx6k - test2/velero-cassandra-data-cassandra-2-k5rbn snapshot.storage.k8s.io/v1/VolumeSnapshotClass: - csi-aws-vsc snapshot.storage.k8s.io/v1/VolumeSnapshotContent: - snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401 - snapcontent-9feaa340-a009-4290-808d-05b030a91072 - snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8 v1/ConfigMap: - test2/kube-root-ca.crt - test2/openshift-service-ca.crt v1/Endpoints: - test2/cassandra v1/Event: - test2/cassandra-0.17c09f55d9832657 - test2/cassandra-0.17c09f5664c1dc76 - test2/cassandra-0.17c09f570fa73c1f - test2/cassandra-0.17c09f57114ffde3 - test2/cassandra-0.17c09f61aa5618e8 - test2/cassandra-0.17c09f61b08f8ab7 - test2/cassandra-0.17c09f61b185a061 - test2/cassandra-1.17c09f62c544f293 - test2/cassandra-1.17c09f634e29dc91 - test2/cassandra-1.17c09f63e9529083 - test2/cassandra-1.17c09f63eaff7dc8 - test2/cassandra-1.17c09f63ff29e827 - test2/cassandra-1.17c09f64055c22d2 - test2/cassandra-1.17c09f640642aa10 - test2/cassandra-2.17c09f65270f8a38 - test2/cassandra-2.17c09f65b2836c64 - test2/cassandra-2.17c09f67340d0ac7 - test2/cassandra-2.17c09f6735bd4461 - test2/cassandra-2.17c09f6870c443de - test2/cassandra-2.17c09f6877ce44b3 - test2/cassandra-2.17c09f6878de3a7f - test2/cassandra-2.17c09f6db84f98f5 - test2/cassandra-data-cassandra-0.17c09f54e9547335 - test2/cassandra-data-cassandra-0.17c09f54eb5d0f9a - test2/cassandra-data-cassandra-0.17c09f54eb5eaeb7 - test2/cassandra-data-cassandra-0.17c09f55ba350fe2 - test2/cassandra-data-cassandra-1.17c09f61d5134475 - test2/cassandra-data-cassandra-1.17c09f61d6fdc837 - test2/cassandra-data-cassandra-1.17c09f61d700ca7f - test2/cassandra-data-cassandra-1.17c09f62a506095b - test2/cassandra-data-cassandra-2.17c09f6437106842 - test2/cassandra-data-cassandra-2.17c09f64389639c7 - test2/cassandra-data-cassandra-2.17c09f6438967fee - test2/cassandra-data-cassandra-2.17c09f6500affef6 - test2/cassandra.17c09f54e954b7b0 - test2/cassandra.17c09f54ea177adc - test2/cassandra.17c09f61d5121f05 - test2/cassandra.17c09f61d5d4d5d3 - test2/cassandra.17c09f64370f737f - test2/cassandra.17c09f6437b10b0d v1/Namespace: - test2 v1/PersistentVolume: - pvc-2d084ccd-24c3-4ae5-863f-0f05a58801c4 - pvc-51740cf1-f13a-4c3e-8777-d67ee2a8ecaa - pvc-60f779f3-cadc-4ea4-873c-1278946a5ece v1/PersistentVolumeClaim: - test2/cassandra-data-cassandra-0 - test2/cassandra-data-cassandra-1 - test2/cassandra-data-cassandra-2 v1/Pod: - test2/cassandra-0 - test2/cassandra-1 - test2/cassandra-2 v1/Secret: - test2/builder-dockercfg-7vl2n - test2/builder-token-n5dsd - test2/default-dockercfg-cjs2m - test2/default-token-vpt8x - test2/deployer-dockercfg-hwjtq - test2/deployer-token-26wdw v1/Service: - test2/cassandra v1/ServiceAccount: - test2/builder - test2/default - test2/deployer Velero-Native Snapshots: <none included>
$ velero describe restore test-restore2 -n openshift-adp --details Name: test-restore2 Namespace: openshift-adp Labels: <none> Annotations: <none> Phase: Completed Total items to be restored: 56 Items restored: 56 Started: 2024-03-27 18:29:41 +0530 IST Completed: 2024-03-27 18:30:01 +0530 IST Warnings: Velero: <none> Cluster: could not restore, CustomResourceDefinition "clusterserviceversions.operators.coreos.com" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, VolumeSnapshotContent "snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, VolumeSnapshotContent "snapcontent-9feaa340-a009-4290-808d-05b030a91072" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, VolumeSnapshotContent "snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8" already exists. Warning: the in-cluster version is different than the backed-up version Namespaces: test2: could not restore, RoleBinding "dedicated-admins-project-dedicated-admins" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, RoleBinding "system:image-pullers" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, ConfigMap "kube-root-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version could not restore, ConfigMap "openshift-service-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin-dedicated-admins: rolebindings.authorization.openshift.io "admin-dedicated-admins" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin-system:serviceaccounts:dedicated-admin: rolebindings.authorization.openshift.io "admin-system:serviceaccounts:dedicated-admin" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/admin: rolebindings.authorization.openshift.io "admin" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/alert-routing-edit-dedicated-admins: rolebindings.authorization.openshift.io "alert-routing-edit-dedicated-admins" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/dedicated-admins-project-dedicated-admins: rolebindings.authorization.openshift.io "dedicated-admins-project-dedicated-admins" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin: rolebindings.authorization.openshift.io "dedicated-admins-project-system:serviceaccounts:dedicated-admin" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:deployers: rolebindings.authorization.openshift.io "system:deployers" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:image-builders: rolebindings.authorization.openshift.io "system:image-builders" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:image-pullers: rolebindings.authorization.openshift.io "system:image-pullers" not found error getting resource from lister for rolebindings.authorization.openshift.io, test2/system:openshift:scc:privileged: rolebindings.authorization.openshift.io "system:openshift:scc:privileged" not found Backup: test-backup2 Namespaces: Included: all namespaces found in the backup Excluded: <none> Resources: Included: * Excluded: nodes, events, events.events.k8s.io, backups.velero.io, restores.velero.io, resticrepositories.velero.io, csinodes.storage.k8s.io, volumeattachments.storage.k8s.io, backuprepositories.velero.io Cluster-scoped: auto Namespace mappings: <none> Label selector: <none> Or label selector: <none> Restore PVs: auto Existing Resource Policy: <none> ItemOperationTimeout: 4h0m0s Preserve Service NodePorts: auto Resource List: apiextensions.k8s.io/v1/CustomResourceDefinition: - clusterserviceversions.operators.coreos.com(failed) apps/v1/ControllerRevision: - test2/cassandra-846b97fbc(created) apps/v1/StatefulSet: - test2/cassandra(created) authorization.openshift.io/v1/RoleBinding: - test2/admin(failed) - test2/admin-dedicated-admins(failed) - test2/admin-system:serviceaccounts:dedicated-admin(failed) - test2/alert-routing-edit-dedicated-admins(failed) - test2/dedicated-admins-project-dedicated-admins(failed) - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin(failed) - test2/system:deployers(failed) - test2/system:image-builders(failed) - test2/system:image-pullers(failed) - test2/system:openshift:scc:privileged(failed) discovery.k8s.io/v1/EndpointSlice: - test2/cassandra-cmhg5(created) operators.coreos.com/v1alpha1/ClusterServiceVersion: - test2/observability-operator.v0.0.28(created) - test2/route-monitor-operator.v0.1.632-df5f264(created) rbac.authorization.k8s.io/v1/RoleBinding: - test2/admin(created) - test2/admin-dedicated-admins(created) - test2/admin-system:serviceaccounts:dedicated-admin(created) - test2/alert-routing-edit-dedicated-admins(created) - test2/dedicated-admins-project-dedicated-admins(failed) - test2/dedicated-admins-project-system:serviceaccounts:dedicated-admin(created) - test2/system:deployers(created) - test2/system:image-builders(created) - test2/system:image-pullers(failed) - test2/system:openshift:scc:privileged(created) snapshot.storage.k8s.io/v1/VolumeSnapshot: - test2/velero-cassandra-data-cassandra-0-4zmqt(created) - test2/velero-cassandra-data-cassandra-1-qjx6k(created) - test2/velero-cassandra-data-cassandra-2-k5rbn(created) snapshot.storage.k8s.io/v1/VolumeSnapshotClass: - csi-aws-vsc(skipped) snapshot.storage.k8s.io/v1/VolumeSnapshotContent: - snapcontent-105fb09c-74cb-4a35-b842-dd169d5ac401(failed) - snapcontent-9feaa340-a009-4290-808d-05b030a91072(failed) - snapcontent-d53fe2cb-1663-430d-978f-f96ccab700b8(failed) v1/ConfigMap: - test2/kube-root-ca.crt(failed) - test2/openshift-service-ca.crt(failed) v1/Endpoints: - test2/cassandra(created) v1/Namespace: - test2(created) v1/PersistentVolume: - pvc-2d084ccd-24c3-4ae5-863f-0f05a58801c4(skipped) - pvc-51740cf1-f13a-4c3e-8777-d67ee2a8ecaa(skipped) - pvc-60f779f3-cadc-4ea4-873c-1278946a5ece(skipped) v1/PersistentVolumeClaim: - test2/cassandra-data-cassandra-0(created) - test2/cassandra-data-cassandra-1(created) - test2/cassandra-data-cassandra-2(created) v1/Pod: - test2/cassandra-0(created) - test2/cassandra-1(created) - test2/cassandra-2(created) v1/Secret: - test2/builder-dockercfg-7vl2n(created) - test2/builder-token-n5dsd(skipped) - test2/default-dockercfg-cjs2m(created) - test2/default-token-vpt8x(skipped) - test2/deployer-dockercfg-hwjtq(created) - test2/deployer-token-26wdw(skipped) v1/Service: - test2/cassandra(created) v1/ServiceAccount: - test2/builder(skipped) - test2/default(skipped) - test2/deployer(skipped)
StatefulSet yaml
$ oc get statefulset -n ocp-cassandra cassandra -o yaml apiVersion: apps/v1 kind: StatefulSet metadata: annotations: openshift.io/backup-registry-hostname: image-registry.openshift-image-registry.svc:5000 openshift.io/backup-server-version: "1.27" openshift.io/restore-registry-hostname: image-registry.openshift-image-registry.svc:5000 openshift.io/restore-server-version: "1.27" creationTimestamp: "2024-03-27T06:24:09Z" generation: 1 labels: app: cassandra velero.io/backup-name: test-backup velero.io/restore-name: test-restore name: cassandra namespace: ocp-cassandra resourceVersion: "352983" uid: 8dd5868d-f67c-413e-8426-0c02c31d10a9 spec: persistentVolumeClaimRetentionPolicy: whenDeleted: Retain whenScaled: Retain podManagementPolicy: OrderedReady replicas: 3 revisionHistoryLimit: 10 selector: matchLabels: app: cassandra serviceName: cassandra template: metadata: creationTimestamp: null labels: app: cassandra spec: containers: - env: - name: MAX_HEAP_SIZE value: 512M - name: HEAP_NEWSIZE value: 512M - name: CASSANDRA_SEEDS value: cassandra-0.cassandra.ocp-cassandra.svc.cluster.local - name: CASSANDRA_CLUSTER_NAME value: Cassandra - name: CASSANDRA_DC value: DC1 - name: CASSANDRA_RACK value: Rack1 - name: CASSANDRA_AUTO_BOOTSTRAP value: "false" - name: CASSANDRA_ENDPOINT_SNITCH value: GossipingPropertyFileSnitch - name: POD_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP image: quay.io/migqe/cassandra:multiarch imagePullPolicy: Always lifecycle: preStop: exec: command: - /bin/sh - -c - nodetool drain name: cassandra ports: - containerPort: 7000 name: intra-node protocol: TCP - containerPort: 7001 name: tls-intra-node protocol: TCP - containerPort: 7199 name: jmx protocol: TCP - containerPort: 9042 name: cql protocol: TCP resources: limits: cpu: 500m memory: 1Gi requests: cpu: 500m memory: 1Gi securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/lib/cassandra name: cassandra-data dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 1800 updateStrategy: rollingUpdate: partition: 0 type: RollingUpdate volumeClaimTemplates: - apiVersion: v1 kind: PersistentVolumeClaim metadata: creationTimestamp: null name: cassandra-data namespace: ocp-cassandra spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi volumeMode: Filesystem status: phase: Pending status: availableReplicas: 1 collisionCount: 0 currentReplicas: 3 currentRevision: cassandra-76bd54848b observedGeneration: 1 readyReplicas: 1 replicas: 3 updateRevision: cassandra-76bd54848b updatedReplicas: 3