Major Scenario -->
In cluster, if multiple filesystems are present with buckets created with those FS and one of the filesystem is unavailable then BLC run is not working for other healthy filesystems.
GPFS Unmount -->
[root@bl5june-21 ~]# mmumount gpfs0
Thu Jun 5 11:11:46 AM CEST 2025: mmumount: Unmounting file systems ...
[root@bl5june-21 ~]#
[root@bl5june-21 ~]# mmlsmount all -L
File system cesSharedRoot is mounted on 3 nodes:
10.0.100.31 bl5june-23
10.0.100.95 bl5june-22
10.0.100.50 bl5june-21
File system fsnew is mounted on 3 nodes:
10.0.100.95 bl5june-22
10.0.100.31 bl5june-23
10.0.100.50 bl5june-21
File system gpfs0 is mounted on 3 nodes:
10.0.100.31 bl5june-23
10.0.100.95 bl5june-22
10.0.100.50 bl5june-21 (internal mount)
[root@bl5june-21 ~]#
Change RUN_TIME and remove crontab from healthy node in order to run crontab from unmounted node -->
[root@bl5june-21 ~]# crontab -l
*/10 * * * * /usr/lpp/mmfs/bin/mmcess3backup /mnt/cesSharedRoot/ces /mnt/cesSharedRoot/ces/mms3-config.json 600
20 11 * * * /usr/lpp/mmfs/bin/dos3lifecycle
[root@bl5june-21 ~]#
[root@bl5june-23 ~]# crontab -l
*/10 * * * * /usr/lpp/mmfs/bin/mmcess3backup /mnt/cesSharedRoot/ces /mnt/cesSharedRoot/ces/mms3-config.json 600
[root@bl5june-23 ~]#
[root@bl5june-22 ~]# crontab -l
*/10 * * * * /usr/lpp/mmfs/bin/mmcess3backup /mnt/cesSharedRoot/ces /mnt/cesSharedRoot/ces/mms3-config.json 600
[root@bl5june-22 ~]#
[root@bl5june-21 ~]# jq . /var/log/noobaa/lifecycle/lifecycle_run_1749115202402.json
{
"running_host": "bl5june-21",
"lifecycle_run_times":
{ "run_lifecycle_start_time": 1749115202402, "list_buckets_start_time": 1749115202422, "list_buckets_end_time": 1749115202423, "list_buckets_took_ms": 1, "process_buckets_start_time": 1749115202423, "create_gpfs_candidates_files_start_time": 1749115202423, "create_gpfs_candidates_files_end_time": 1749115205791, "create_gpfs_candidates_files_took_ms": 3368, "process_buckets_end_time": 1749115205791, "process_buckets_took_ms": 3368, "run_lifecycle_end_time": 1749115205791, "run_lifecycle_took_ms": 3389 },
"total_stats":
{ "num_objects_deleted": 0, "num_objects_delete_failed": 0, "objects_delete_errors": [], "num_mpu_aborted": 0, "num_mpu_abort_failed": 0, "mpu_abort_errors": [] },
"state":
{ "is_finished": false },
"buckets_statuses": {},
"errors": [
"create_candidates_file_by_gpfs_ilm_policy failed with error Error: Command failed: mmapplypolicy /mnt/gpfs0 -P /var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1
749115202402 -f /var/log/noobaa/lifecycle/lifecycle_ilm_candidates -I defer\nmmapplypolicy: /mnt/gpfs0 is not a GPFS file system object.\nmmapplypolicy: Command failed. Examine previous error message
s to determine cause.\n",
"create_candidates_file_by_gpfs_ilm_policy failed with error Error: Command failed: mmapplypolicy /mnt/gpfs0 -P /var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1
749115202402 -f /var/log/noobaa/lifecycle/lifecycle_ilm_candidates -I defer\nmmapplypolicy: /mnt/gpfs0 is not a GPFS file system object.\nmmapplypolicy: Command failed. Examine previous error message
s to determine cause.\n",
"create_candidates_file_by_gpfs_ilm_policy failed with error Error: Command failed: mmapplypolicy /mnt/gpfs0 -P /var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1
749115202402 -f /var/log/noobaa/lifecycle/lifecycle_ilm_candidates -I defer\nmmapplypolicy: /mnt/gpfs0 is not a GPFS file system object.\nmmapplypolicy: Command failed. Examine previous error message
s to determine cause.\n"
]
}
[root@bl5june-21 ~]#
<-- Build Details -->
[root@bl5june-21 ~]# rpm -qi gpfs.mms3-5.2.3-1.el9.x86_64
Name : gpfs.mms3
Install Date: Thu 05 Jun 2025 09:07:21 AM CEST
Signature : RSA/SHA256, Wed 04 Jun 2025 11:56:16 PM CEST, Key ID e1ecfb69c4c9e95d
Source RPM : gpfs.mms3-5.2.3-1.el9.src.rpm
Build Date : Wed 04 Jun 2025 11:39:35 PM CEST
[root@bl5june-21 ~]#
[root@bl5june-21 ~]# rpm -qi noobaa-core-5.18.4-20250527.el9.x86_64
Name : noobaa-core
Install Date: Thu 05 Jun 2025 09:07:53 AM CEST
Signature : RSA/SHA256, Thu 29 May 2025 12:12:49 AM CEST, Key ID e1ecfb69c4c9e95d
Source RPM : noobaa-core-5.18.4-20250527.el9.src.rpm
Build Date : Tue 27 May 2025 09:26:11 AM CEST
[root@bl5june-21 ~]#
