load_nsfs_nc_config.setting config.NSFS_NC_CONF_DIR /mnt/cesSharedRoot/ces/s3-config nsfs: config_dir_path=/mnt/cesSharedRoot/ces/s3-config nsfs: config.json= { ENDPOINT_FORKS: 2, ENDPOINT_PORT: 6001, ENDPOINT_SSL_PORT: 6443, LOG_TO_SYSLOG_ENABLED: true, ALLOW_HTTP: false, NSFS_CALCULATE_MD5: false, UV_THREADPOOL_SIZE: 16, GPFS_DL_PATH: '/usr/lpp/mmfs/lib/libgpfs.so', LOG_TO_STDERR_ENABLED: true, NC_DISABLE_ACCESS_CHECK: false, NOOBAA_LOG_LEVEL: 'default', NSFS_NC_STORAGE_BACKEND: 'GPFS', NSFS_NC_CONFIG_DIR_BACKEND: 'GPFS', NSFS_DIR_CACHE_MAX_DIR_SIZE: 536870912, NSFS_DIR_CACHE_MAX_TOTAL_SIZE: 1073741824, NSFS_GLACIER_ENABLED: false, NSFS_GLACIER_LOGS_ENABLED: false, NC_MASTER_KEYS_GET_EXECUTABLE: '/usr/lpp/mmfs/bin/cess3_key_get', NC_MASTER_KEYS_PUT_EXECUTABLE: '/usr/lpp/mmfs/bin/cess3_key_put', NC_MASTER_KEYS_STORE_TYPE: 'executable', NC_DISABLE_POSIX_MODE_ACCESS_CHECK: true, CONFIG_FILE: '', NOTIFICATION_LOG_DIR: '', NOTIFICATION_FREQUENCY_IN_MINUTES: 0, CONFIG_KV: [], NC_LIFECYCLE_TIMEOUT_MS: 28800000, NC_LIFECYCLE_RUN_TIME: '11:23', NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS: 10, NC_LIFECYCLE_TZ: 'LOCAL', NC_LIFECYCLE_LOGS_DIR: '/var/log/noobaa/lifecycle' } nsfs: merged config.json= { ENDPOINT_FORKS: 2, ENDPOINT_PORT: 6001, ENDPOINT_SSL_PORT: 6443, LOG_TO_SYSLOG_ENABLED: true, ALLOW_HTTP: false, NSFS_CALCULATE_MD5: false, UV_THREADPOOL_SIZE: 16, GPFS_DL_PATH: '/usr/lpp/mmfs/lib/libgpfs.so', LOG_TO_STDERR_ENABLED: true, NC_DISABLE_ACCESS_CHECK: false, NOOBAA_LOG_LEVEL: 'default', NSFS_NC_STORAGE_BACKEND: 'GPFS', NSFS_NC_CONFIG_DIR_BACKEND: 'GPFS', NSFS_DIR_CACHE_MAX_DIR_SIZE: 536870912, NSFS_DIR_CACHE_MAX_TOTAL_SIZE: 1073741824, NSFS_GLACIER_ENABLED: false, NSFS_GLACIER_LOGS_ENABLED: false, NC_MASTER_KEYS_GET_EXECUTABLE: '/usr/lpp/mmfs/bin/cess3_key_get', NC_MASTER_KEYS_PUT_EXECUTABLE: '/usr/lpp/mmfs/bin/cess3_key_put', NC_MASTER_KEYS_STORE_TYPE: 'executable', NC_DISABLE_POSIX_MODE_ACCESS_CHECK: true, CONFIG_FILE: '', NOTIFICATION_LOG_DIR: '', NOTIFICATION_FREQUENCY_IN_MINUTES: 0, CONFIG_KV: [], NC_LIFECYCLE_TIMEOUT_MS: 28800000, NC_LIFECYCLE_RUN_TIME: '11:23', NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS: 10, NC_LIFECYCLE_TZ: 'LOCAL', NC_LIFECYCLE_LOGS_DIR: '/var/log/noobaa/lifecycle' } 2025-05-26 11:30:03.113056 [PID-123277/TID-123277] FS::GPFS GPFS_DL_PATH=/usr/lpp/mmfs/lib/libgpfs.so 2025-05-26 11:30:03.113141 [PID-123277/TID-123277] FS::GPFS found GPFS lib file GPFS_DL_PATH=/usr/lpp/mmfs/lib/libgpfs.so May-26 11:30:03.451 [noobaa-cli/123277]  [LOG] CONSOLE:: detect_fips_mode: found /proc/sys/crypto/fips_enabled with value 0 (node:123277) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead. (Use `node --trace-deprecation ...` to show where the warning was created) 2025-05-26 11:30:03.717581 [PID-123277/TID-123277] [L1] FS::set_debug_level 5 2025-05-26 11:30:03.718303 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config 2025-05-26 11:30:03.718469 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.718692 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.719154 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config took: 0.406345 ms May-26 11:30:03.719 [noobaa-cli/123277]  [LOG] CONSOLE:: read_rand_seed: reading 32 bytes from /dev/urandom ... 2025-05-26 11:30:03.719893 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config _stat_res.st_ino=20739 _stat_res.st_size=8192 May-26 11:30:03.720 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config 2025-05-26 11:30:03.720245 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets 2025-05-26 11:30:03.720326 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.720572 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.720685 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets took: 0.057361 ms May-26 11:30:03.732 [noobaa-cli/123277]  [LOG] CONSOLE:: read_rand_seed: got 32 bytes from /dev/urandom, total 32 ... May-26 11:30:03.732 [noobaa-cli/123277]  [LOG] CONSOLE:: read_rand_seed: closing fd ... 2025-05-26 11:30:03.732867 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/buckets _stat_res.st_ino=19474 _stat_res.st_size=8192 May-26 11:30:03.732 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/buckets 2025-05-26 11:30:03.733115 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/accounts_by_name 2025-05-26 11:30:03.733178 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/accounts_by_name _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.733457 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/accounts_by_name _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.733577 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/accounts_by_name took: 0.06512 ms May-26 11:30:03.733 [noobaa-cli/123277]  [LOG] CONSOLE:: init_rand_seed: seeding with 32 bytes 2025-05-26 11:30:03.733817 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/accounts_by_name _stat_res.st_ino=19478 _stat_res.st_size=4096 May-26 11:30:03.733 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/accounts_by_name 2025-05-26 11:30:03.734034 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities 2025-05-26 11:30:03.734096 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.734360 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.734480 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities took: 0.059967 ms 2025-05-26 11:30:03.734558 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/identities _stat_res.st_ino=19480 _stat_res.st_size=4096 May-26 11:30:03.734 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/identities 2025-05-26 11:30:03.734837 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/access_keys 2025-05-26 11:30:03.734901 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/access_keys _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.735163 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/access_keys _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.735266 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/access_keys took: 0.051921 ms 2025-05-26 11:30:03.735349 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/access_keys _stat_res.st_ino=19471 _stat_res.st_size=4096 May-26 11:30:03.735 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/access_keys 2025-05-26 11:30:03.735624 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/connections 2025-05-26 11:30:03.735688 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/connections _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.735928 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/connections _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.736032 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/connections took: 0.05204 ms 2025-05-26 11:30:03.736100 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/connections _stat_res.st_ino=19469 _stat_res.st_size=4096 May-26 11:30:03.736 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/connections 2025-05-26 11:30:03.736673 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/ _mode=448 2025-05-26 11:30:03.736764 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.737007 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.737069 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 took: 0.013969 ms 2025-05-26 11:30:03.737143 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/ _mode=448 error.Message()=File exists 2025-05-26 11:30:03.737279 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var _mode=448 2025-05-26 11:30:03.737336 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.737632 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.737691 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 took: 0.003843 ms 2025-05-26 11:30:03.737760 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var _mode=448 error.Message()=File exists 2025-05-26 11:30:03.737846 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log _mode=448 2025-05-26 11:30:03.737901 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.738136 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.738186 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 took: 0.003779 ms 2025-05-26 11:30:03.738249 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log _mode=448 error.Message()=File exists 2025-05-26 11:30:03.738328 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa _mode=448 2025-05-26 11:30:03.738383 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.738639 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.738690 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 took: 0.004187 ms 2025-05-26 11:30:03.738761 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa _mode=448 error.Message()=File exists 2025-05-26 11:30:03.738837 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 2025-05-26 11:30:03.738907 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.739074 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.739187 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 took: 0.066353 ms 2025-05-26 11:30:03.739258 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 error.Message()=File exists 2025-05-26 11:30:03.739340 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Fsync _path=/var/log/noobaa/lifecycle 2025-05-26 11:30:03.739401 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:03.739523 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.741168 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle took: 1.56008 ms 2025-05-26 11:30:03.741513 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Fsync _path=/var/log/noobaa/lifecycle 2025-05-26 11:30:03.741823 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle 2025-05-26 11:30:03.742025 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.742228 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.742625 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle took: 0.225337 ms 2025-05-26 11:30:03.742855 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle _stat_res.st_ino=19488 _stat_res.st_size=4096 May-26 11:30:03.743 [noobaa-cli/123277]  [L1] core.sdk.config_fs:: create_dir_if_missing: config dir exists: /mnt/cesSharedRoot/ces/s3-config/lifecycle 2025-05-26 11:30:03.743747 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: FileOpen _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _flags=577 _mode=438 2025-05-26 11:30:03.743968 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _flags=577 _mode=438 _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.744177 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _flags=577 _mode=438 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.751893 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _flags=577 _mode=438 took: 7.55041 ms 2025-05-26 11:30:03.752208 [PID-123277/TID-123277] [L1] FS::FileOpen::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock 2025-05-26 11:30:03.752634 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: FileFcntlLock_wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock 2025-05-26 11:30:03.752846 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: FileFcntlLock_wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.753091 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: FileFcntlLock_wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.753492 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: FileFcntlLock_wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock took: 0.238259 ms 2025-05-26 11:30:03.753700 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined FileFcntlLock_wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock May-26 11:30:03.754 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: run_lifecycle_under_lock acquired lock - verifying May-26 11:30:03.754 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: _should_lifecycle_run should_run true May-26 11:30:03.754 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: run_lifecycle_under_lock acquired lock - start lifecycle May-26 11:30:03.757 [noobaa-cli/123277] [EVENT]{"timestamp":"2025-05-26T09:30:03.756Z","host":"21may-11","event":{"code":"noobaa_lifecycle_worker_started","message":"NooBaa Lifecycle worker run started.","description":"NooBaa Lifecycle worker run started.","entity_type":"NODE","event_type":"INFO","scope":"NODE","severity":"INFO","state":"HEALTHY","pid":123277}} 2025-05-26 11:30:03.758723 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/system.json 2025-05-26 11:30:03.758988 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/system.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.759212 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/system.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.759609 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/system.json took: 0.233389 ms 2025-05-26 11:30:03.759997 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/system.json May-26 11:30:03.760 [noobaa-cli/123277]  [L2] core.util.os_utils:: promise exec systemctl show -p ActiveState --value noobaa false 2025-05-26 11:30:03.783935 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readdir _path=/mnt/cesSharedRoot/ces/s3-config/buckets 2025-05-26 11:30:03.784063 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Readdir _path=/mnt/cesSharedRoot/ces/s3-config/buckets _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:03.784268 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Readdir _path=/mnt/cesSharedRoot/ces/s3-config/buckets _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:03.784384 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Readdir _path=/mnt/cesSharedRoot/ces/s3-config/buckets took: 0.06055 ms 2025-05-26 11:30:03.784527 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readdir _path=/mnt/cesSharedRoot/ces/s3-config/buckets May-26 11:30:03.785 [noobaa-cli/123277]  [L2] core.util.os_utils:: promise exec mmlsfs all -T -Y false May-26 11:30:04.018 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: get_mount_points fs_list res mmlsfs::HEADER:version:reserved:reserved:deviceName:fieldName:data:remarks: mmlsfs::0:1:::cesSharedRoot:defaultMountPoint:%2Fmnt%2FcesSharedRoot:: mmlsfs::0:1:::gpfs0:defaultMountPoint:%2Fmnt%2Fgpfs0:: 2025-05-26 11:30:04.018913 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:04.019096 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:04.019479 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.019663 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json took: 0.064582 ms 2025-05-26 11:30:04.019758 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json May-26 11:30:04.020 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: find_mount_point_by_bucket_path bucket_path=/mnt/gpfs0/account7000/bucket7000 mount_point_path={ '/mnt/cesSharedRoot': '', '/mnt/gpfs0': '' } 2025-05-26 11:30:04.020854 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json 2025-05-26 11:30:04.020982 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:04.021382 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.021565 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json took: 0.055441 ms 2025-05-26 11:30:04.021701 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json May-26 11:30:04.022 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: find_mount_point_by_bucket_path bucket_path=/mnt/gpfs0/account7000/bucket123 mount_point_path={ '/mnt/cesSharedRoot': '', '/mnt/gpfs0': '' } 2025-05-26 11:30:04.023062 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/ _mode=448 2025-05-26 11:30:04.023217 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.023595 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.023688 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 took: 0.024732 ms 2025-05-26 11:30:04.023862 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/ _mode=448 error.Message()=File exists 2025-05-26 11:30:04.024055 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var _mode=448 2025-05-26 11:30:04.024159 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.024523 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.024663 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 took: 0.018753 ms 2025-05-26 11:30:04.024737 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var _mode=448 error.Message()=File exists 2025-05-26 11:30:04.024926 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log _mode=448 2025-05-26 11:30:04.025051 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.025182 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.025240 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 took: 0.00347 ms 2025-05-26 11:30:04.025418 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log _mode=448 error.Message()=File exists 2025-05-26 11:30:04.025627 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa _mode=448 2025-05-26 11:30:04.025720 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.026036 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.026108 [PID-123277/TID-123294] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 took: 0.00428 ms 2025-05-26 11:30:04.026180 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa _mode=448 error.Message()=File exists 2025-05-26 11:30:04.026266 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 2025-05-26 11:30:04.026326 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.026529 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.026585 [PID-123277/TID-123295] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 took: 0.004641 ms 2025-05-26 11:30:04.026646 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 error.Message()=File exists 2025-05-26 11:30:04.026746 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _mode=448 2025-05-26 11:30:04.026808 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.027005 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.027098 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _mode=448 took: 0.0442 ms 2025-05-26 11:30:04.027166 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _mode=448 2025-05-26 11:30:04.027232 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies 2025-05-26 11:30:04.027291 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.027513 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.028525 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies took: 0.960302 ms 2025-05-26 11:30:04.028611 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies 2025-05-26 11:30:04.028693 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/ _mode=448 2025-05-26 11:30:04.028753 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.028934 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.028984 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Mkdir _path=/ _mode=448 took: 0.002945 ms 2025-05-26 11:30:04.029052 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/ _mode=448 error.Message()=File exists 2025-05-26 11:30:04.029133 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var _mode=448 2025-05-26 11:30:04.029218 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.029277 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.029301 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Mkdir _path=/var _mode=448 took: 0.004114 ms 2025-05-26 11:30:04.029356 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var _mode=448 error.Message()=File exists 2025-05-26 11:30:04.029463 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log _mode=448 2025-05-26 11:30:04.029528 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.029588 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.029634 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log _mode=448 took: 0.003298 ms 2025-05-26 11:30:04.029695 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log _mode=448 error.Message()=File exists 2025-05-26 11:30:04.029768 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa _mode=448 2025-05-26 11:30:04.029823 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.029876 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.029923 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa _mode=448 took: 0.003143 ms 2025-05-26 11:30:04.029981 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa _mode=448 error.Message()=File exists 2025-05-26 11:30:04.030076 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 2025-05-26 11:30:04.030138 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.030311 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.030358 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 took: 0.003762 ms 2025-05-26 11:30:04.030426 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Mkdir _path=/var/log/noobaa/lifecycle _mode=448 error.Message()=File exists 2025-05-26 11:30:04.030527 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _mode=448 2025-05-26 11:30:04.030583 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _mode=448 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.030639 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _mode=448 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.030711 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _mode=448 took: 0.028701 ms 2025-05-26 11:30:04.030765 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Mkdir _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _mode=448 2025-05-26 11:30:04.030837 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates 2025-05-26 11:30:04.030890 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.031076 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.032145 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates took: 1.02273 ms 2025-05-26 11:30:04.032226 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Fsync _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates 2025-05-26 11:30:04.032466 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 2025-05-26 11:30:04.032550 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.032613 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.032669 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 took: 0.009712 ms 2025-05-26 11:30:04.032732 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: Stat _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 error.Message()=No such file or directory 2025-05-26 11:30:04.032853 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _len=568 _mode=432 2025-05-26 11:30:04.032914 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _len=568 _mode=432 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:04.032989 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _len=568 _mode=432 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:04.033119 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _len=568 _mode=432 took: 0.081699 ms 2025-05-26 11:30:04.033184 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Writefile _path=/var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 _len=568 _mode=432 May-26 11:30:04.033 [noobaa-cli/123277]  [L2] core.util.os_utils:: promise exec mmapplypolicy /mnt/gpfs0 -P /var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757 -f /var/log/noobaa/lifecycle/lifecycle_ilm_candidates -I defer false May-26 11:30:04.115 [noobaa-cli/123277]  [LOG] CONSOLE:: generate_entropy: entropy_avail 256 May-26 11:30:04.127 [noobaa-cli/123277]  [LOG] CONSOLE:: generate_entropy: adding entropy: dd if=/dev/vda bs=1048576 count=32 skip=21366 | md5sum May-26 11:30:04.212 [noobaa-cli/123277]  [LOG] CONSOLE:: init_rand_seed: done May-26 11:30:05.278 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: create_candidates_file_by_gpfs_ilm_policy mmapplypolicy res [I] IscanBuckets set to 7. [I] GPFS Current Data Pool Utilization in KB and % Pool_Name KB_Occupied KB_Total Percent_Occupied data 1917696 20971520 9.144287109% system 0 0 0.000000000% (no user data) [I] 4188 of 133376 inodes used: 3.139995%. [I] Loaded policy rules from /var/log/noobaa/lifecycle/lifecycle_ilm_policies/noobaa_ilm_policy_%2Fmnt%2Fgpfs0_1748251803757. Evaluating policy rules with CURRENT_TIMESTAMP = 2025-05-26@09:30:04 UTC Parsed 1 policy rules. RULE 'bucket123_Expiration Rule_1748251803757' LIST 'bucket123_Expiration Rule_1748251803757' WHERE PATH_NAME LIKE '/mnt/gpfs0/account7000/bucket123/%' AND PATH_NAME NOT LIKE '/mnt/gpfs0/account7000/bucket123/.noobaa_nsfs%/%' AND PATH_NAME NOT LIKE '/mnt/gpfs0/account7000/bucket123/.versions/%' AND PATH_NAME NOT LIKE '/mnt/gpfs0/account7000/bucket123/%/.versions/%' AND PATH_NAME LIKE '/mnt/gpfs0/account7000/bucket123/file%' [I] 2025-05-26@09:30:04.672 Directory entries scanned: 0. [I] 2025-05-26@09:30:04.685 Directory entries scanned: 159. [I] Directories scan: 30 files, 129 directories, 0 other objects, 0 'skipped' files and/or errors. [I] 2025-05-26@09:30:04.686 Parallel-piped sort and policy evaluation. 0 files scanned. [I] 2025-05-26@09:30:05.100 Parallel-piped sort and policy evaluation. 159 files scanned. [I] 2025-05-26@09:30:05.115 Piped sorting and candidate file choosing. 0 records scanned. [I] 2025-05-26@09:30:05.150 Piped sorting and candidate file choosing. 0 records scanned. [I] Summary of Rule Applicability and File Choices: Rule# Hit_Cnt KB_Hit Chosen KB_Chosen KB_Ill Rule 0 0 0 0 0 0 RULE 'bucket123_Expiration Rule_1748251803757' LIST 'bucket123_Expiration Rule_1748251803757' WHERE(.) [I] Filesystem objects with no applicable rules: 147. Predicted Data Pool Utilization in KB and %: Pool_Name KB_Occupied KB_Total Percent_Occupied data 1917952 20971520 9.145507812% system 0 0 0.000000000% (no user data) [I] 2025-05-26@09:30:05.170 Policy execution. 0 files dispatched. [I] 2025-05-26@09:30:05.174 Policy execution. 0 files dispatched. [I] A total of 0 files have been migrated, deleted or processed by an EXTERNAL EXEC/script; 0 'skipped' files and/or errors. 2025-05-26 11:30:05.279547 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:05.279648 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.279676 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json 2025-05-26 11:30:05.279795 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.279972 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.280128 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.280188 [PID-123277/TID-123297] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json took: 0.043326 ms 2025-05-26 11:30:05.280268 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket123.json 2025-05-26 11:30:05.280358 [PID-123277/TID-123296] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json took: 0.056282 ms 2025-05-26 11:30:05.280615 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.280664 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:05.280714 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.280772 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.280813 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.280865 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.280925 [PID-123277/TID-123299] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json took: 0.038867 ms 2025-05-26 11:30:05.280961 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.281030 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.281107 [PID-123277/TID-123298] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json took: 0.031033 ms May-26 11:30:05.282 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: nc_lifecycle.process_rule: start bucket name: bucket7000 rule { filter: { prefix: 'part' }, id: 'abort-multipart-days-file-prefix', status: 'Enabled', abort_incomplete_multipart_upload: { days_after_initiation: 1 } } index 0 May-26 11:30:05.282 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: nc_lifecycle.process_rule: processing rule: bucket7000 (bucket id: 682ead3e5a333d4395fafc59 ) rule { filter: { prefix: 'part' }, id: 'abort-multipart-days-file-prefix', status: 'Enabled', abort_incomplete_multipart_upload: { days_after_initiation: 1 } } May-26 11:30:05.283 [noobaa-cli/123277]  [L0] core.sdk.bucketspace_fs:: BucketSpaceFS.read_bucket_sdk_info: bucket name bucket7000 2025-05-26 11:30:05.283542 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:05.283606 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.283616 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.283717 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.283792 [PID-123277/TID-123284] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json took: 0.029859 ms May-26 11:30:05.283 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: nc_lifecycle.process_rule: start bucket name: bucket123 rule { filter: { prefix: 'file' }, id: 'Expiration Rule', status: 'Enabled', expiration: { date: 1657584000000 } } index 0 May-26 11:30:05.284 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: nc_lifecycle.process_rule: processing rule: bucket123 (bucket id: 6833f69ae372547586a14189 ) rule { filter: { prefix: 'file' }, id: 'Expiration Rule', status: 'Enabled', expiration: { date: 1657584000000 } } May-26 11:30:05.284 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: parse_candidates_from_gpfs_ilm_policy rule_state=[object Object] state_type=expire, currently on gpfs ilm flow - we support only expiration rule May-26 11:30:05.284 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: parse_candidates_from_gpfs_ilm_policy bucket_name=bucket123, rule_id Expiration Rule, existing rule_state={} 2025-05-26 11:30:05.284788 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: FileOpen _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates/list.bucket123_Expiration Rule_1748251803757 _flags=2 _mode=438 2025-05-26 11:30:05.284843 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:05.284858 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates/list.bucket123_Expiration Rule_1748251803757 _flags=2 _mode=438 _uid=0 _gid=0 _backend= supplemental_groups= 2025-05-26 11:30:05.285106 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates/list.bucket123_Expiration Rule_1748251803757 _flags=2 _mode=438 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.285166 [PID-123277/TID-123285] [L1] FS::FSWorker::Execute: FileOpen _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates/list.bucket123_Expiration Rule_1748251803757 _flags=2 _mode=438 took: 0.01279 ms 2025-05-26 11:30:05.287783 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/gpfs0/account7000/bucket7000 2025-05-26 11:30:05.287840 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: FileOpen _path=/var/log/noobaa/lifecycle/lifecycle_ilm_candidates/list.bucket123_Expiration Rule_1748251803757 _flags=2 _mode=438 error.Message()=No such file or directory 2025-05-26 11:30:05.287892 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/gpfs0/account7000/bucket7000 _uid=0 _gid=0 _backend=GPFS supplemental_groups= May-26 11:30:05.287 [noobaa-cli/123277]  [L2] core.manage_nsfs.nc_lifecycle:: parse_candidates_from_gpfs_ilm_policy ilm_candidates_file_exists does not exist, no candidates to delete 2025-05-26 11:30:05.288105 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/gpfs0/account7000/bucket7000 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.293567 [PID-123277/TID-123286] [L1] FS::FSWorker::Execute: Stat _path=/mnt/gpfs0/account7000/bucket7000 took: 5.41477 ms 2025-05-26 11:30:05.293678 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/gpfs0/account7000/bucket7000 _stat_res.st_ino=16396 _stat_res.st_size=4096 2025-05-26 11:30:05.294124 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.294189 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.294362 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.294468 [PID-123277/TID-123287] [L1] FS::FSWorker::Execute: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json took: 0.07785 ms 2025-05-26 11:30:05.294499 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: Readfile _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json May-26 11:30:05.294 [noobaa-cli/123277]  [L1] core.manage_nsfs.nc_master_key_manager:: _init_from_exec: calling config.NC_MASTER_KEYS_GET_EXECUTABLE script May-26 11:30:05.294 [noobaa-cli/123277]  [L2] core.util.os_utils:: promise exec /usr/lpp/mmfs/bin/cess3_key_get master_keys false May-26 11:30:05.375 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_master_key_manager:: init_from_exec: get master keys response status=OK, version=2 May-26 11:30:05.375 [noobaa-cli/123277]  [L1] core.manage_nsfs.nc_master_key_manager:: _set_keys: master_key_manager updated successfully! May-26 11:30:05.375 [noobaa-cli/123277]  [L1] core.manage_nsfs.nc_master_key_manager:: _init_from_exec: updating this.last_init_time 1748251805375 May-26 11:30:05.376 [noobaa-cli/123277]  [L2] core.sdk.config_fs:: stat_account_config_file_by_identity: will try to stat by identity (identities path) 2025-05-26 11:30:05.376423 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json 2025-05-26 11:30:05.376547 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.376717 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.376813 [PID-123277/TID-123288] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json took: 0.075142 ms 2025-05-26 11:30:05.376851 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/identities/682ead361c413defa8e3e596/identity.json _stat_res.st_ino=65349 _stat_res.st_size=448 2025-05-26 11:30:05.377088 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json 2025-05-26 11:30:05.377139 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.377295 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.377368 [PID-123277/TID-123289] [L1] FS::FSWorker::Execute: Stat _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json took: 0.051689 ms 2025-05-26 11:30:05.377402 [PID-123277/TID-123277] [L1] FS::Stat::OnOK: _path=/mnt/cesSharedRoot/ces/s3-config/buckets/bucket7000.json _stat_res.st_ino=9999 _stat_res.st_size=463 May-26 11:30:05.377 [noobaa-cli/123277]  [L1] core.sdk.object_sdk:: _load_bucket_namespace { _id: '682ead3e5a333d4395fafc59', name: SENSITIVE-f90153357ffff876, owner_account: { id: '682ead361c413defa8e3e596', email: SENSITIVE-993063145e5ca5ea }, tag: [], versioning: 'DISABLED', creation_date: '2025-05-22T04:51:10.830Z', path: '/mnt/gpfs0/account7000/bucket7000', should_create_underlying_storage: false, fs_backend: 'GPFS', lifecycle_configuration_rules: [ { filter: { prefix: 'part' }, id: 'abort-multipart-days-file-prefix', status: 'Enabled', abort_incomplete_multipart_upload: { days_after_initiation: 1 } } ], namespace: { read_resources: [ { resource: { fs_root_path: '', fs_backend: 'GPFS' }, path: '/mnt/gpfs0/account7000/bucket7000' } ], write_resource: { resource: { fs_root_path: '', fs_backend: 'GPFS' }, path: '/mnt/gpfs0/account7000/bucket7000' }, should_create_underlying_storage: false }, bucket_info: { versioning: 'DISABLED', logging: undefined }, bucket_owner: SENSITIVE-993063145e5ca5ea, supported_storage_classes: [ 'STANDARD' ], stat: { dev: 43, ino: 9999, mode: 33200, nlink: 1, uid: 0, gid: 0, rdev: 0, size: 463, blksize: 1048576, blocks: 16, atimeMs: 1748169521721.478, ctimeMs: 1747910244767.802, mtimeMs: 1747910244744.8398, birthtimeMs: 1747910244767.802, atime: 2025-05-25T10:38:41.721Z, mtime: 2025-05-22T10:37:24.745Z, ctime: 2025-05-22T10:37:24.768Z, birthtime: 2025-05-22T10:37:24.768Z, atimeNsBigint: 1748169521721478144n, ctimeNsBigint: 1747910244767802112n, mtimeNsBigint: 1747910244744839936n, xattr: { 'security.selinux': 'system_u:object_r:unlabeled_t:s0\x00' } } } May-26 11:30:05.378 [noobaa-cli/123277]  [L0] core.sdk.namespace_fs:: NamespaceFS: buffers_pool [ BufferPool.get_buffer: sem value: 2097152 waiting_value: 0 buffers length: 0, BufferPool.get_buffer: sem value: 33554432 waiting_value: 0 buffers length: 0, BufferPool.get_buffer: sem value: 143654912 waiting_value: 0 buffers length: 0, BufferPool.get_buffer: sem value: 1291845632 waiting_value: 0 buffers length: 0 ] 2025-05-26 11:30:05.378991 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: DirOpen _path=/mnt/gpfs0/account7000/bucket7000/.noobaa-nsfs_682ead3e5a333d4395fafc59/multipart-uploads 2025-05-26 11:30:05.379049 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: DirOpen _path=/mnt/gpfs0/account7000/bucket7000/.noobaa-nsfs_682ead3e5a333d4395fafc59/multipart-uploads _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.379211 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: DirOpen _path=/mnt/gpfs0/account7000/bucket7000/.noobaa-nsfs_682ead3e5a333d4395fafc59/multipart-uploads _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.379270 [PID-123277/TID-123290] [L1] FS::FSWorker::Execute: DirOpen _path=/mnt/gpfs0/account7000/bucket7000/.noobaa-nsfs_682ead3e5a333d4395fafc59/multipart-uploads took: 0.037137 ms 2025-05-26 11:30:05.379311 [PID-123277/TID-123277] [L1] FS::FSWorker::OnError: DirOpen _path=/mnt/gpfs0/account7000/bucket7000/.noobaa-nsfs_682ead3e5a333d4395fafc59/multipart-uploads error.Message()=No such file or directory 2025-05-26 11:30:05.379672 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Writefile _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle.timestamp _len=24 _mode=438 2025-05-26 11:30:05.379728 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Writefile _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle.timestamp _len=24 _mode=438 _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.379906 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Writefile _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle.timestamp _len=24 _mode=438 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.382601 [PID-123277/TID-123291] [L1] FS::FSWorker::Execute: Writefile _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle.timestamp _len=24 _mode=438 took: 2.66942 ms 2025-05-26 11:30:05.382651 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Writefile _path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle.timestamp _len=24 _mode=438 2025-05-26 11:30:05.382805 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_run_1748251803757.json _len=1922 _mode=432 2025-05-26 11:30:05.382859 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_run_1748251803757.json _len=1922 _mode=432 _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.383009 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_run_1748251803757.json _len=1922 _mode=432 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.383093 [PID-123277/TID-123292] [L1] FS::FSWorker::Execute: Writefile _path=/var/log/noobaa/lifecycle/lifecycle_run_1748251803757.json _len=1922 _mode=432 took: 0.061841 ms 2025-05-26 11:30:05.383125 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined Writefile _path=/var/log/noobaa/lifecycle/lifecycle_run_1748251803757.json _len=1922 _mode=432 May-26 11:30:05.383 [noobaa-cli/123277]  [L0] core.manage_nsfs.nc_lifecycle:: run_lifecycle_under_lock done lifecycle - released lock 2025-05-26 11:30:05.383206 [PID-123277/TID-123277] [L1] FS::FSWorker::Begin: FileClose _wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _wrap->_fd=17 2025-05-26 11:30:05.383244 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: FileClose _wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _wrap->_fd=17 _uid=0 _gid=0 _backend=GPFS supplemental_groups= 2025-05-26 11:30:05.383282 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: FileClose _wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _wrap->_fd=17 _uid=0 _gid=0 geteuid()=0 getegid()=0 getuid()=0 getgid()=0 new_supplemental_groups=0 2025-05-26 11:30:05.383357 [PID-123277/TID-123293] [L1] FS::FSWorker::Execute: FileClose _wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _wrap->_fd=17 took: 0.055863 ms 2025-05-26 11:30:05.383387 [PID-123277/TID-123277] [L1] FS::FSWorker::OnOK: undefined FileClose _wrap->_path=/mnt/cesSharedRoot/ces/s3-config/lifecycle/lifecycle_cluster.lock _wrap->_fd=17 May-26 11:30:05.383 [noobaa-cli/123277] [EVENT]{"timestamp":"2025-05-26T09:30:05.383Z","host":"21may-11","event":{"code":"noobaa_lifecycle_worker_finished_successfully","message":"NooBaa Lifecycle worker run finished successfully.","description":"NooBaa Lifecycle worker finished successfully.","entity_type":"NODE","event_type":"INFO","scope":"NODE","severity":"INFO","state":"HEALTHY","pid":123277}} { "response": { "code": "LifecycleSuccessful", "message": "Lifecycle worker run finished successfully", "reply": { "running_host": "21may-11", "lifecycle_run_times": { "run_lifecycle_start_time": 1748251803757, "list_buckets_start_time": 1748251803783, "list_buckets_end_time": 1748251803784, "list_buckets_took_ms": 1, "process_buckets_start_time": 1748251803784, "create_gpfs_candidates_files_start_time": 1748251803784, "create_gpfs_candidates_files_end_time": 1748251805278, "create_gpfs_candidates_files_took_ms": 1494, "process_buckets_end_time": 1748251805379, "process_buckets_took_ms": 1595, "run_lifecycle_end_time": 1748251805379, "run_lifecycle_took_ms": 1622 }, "total_stats": { "num_objects_deleted": 0, "num_objects_delete_failed": 0, "objects_delete_errors": [], "num_mpu_aborted": 0, "num_mpu_abort_failed": 0, "mpu_abort_errors": [] }, "state": { "is_finished": true }, "buckets_statuses": { "bucket7000": { "bucket_process_times": { "process_bucket_start_time": 1748251805279, "process_bucket_end_time": 1748251805379, "process_bucket_took_ms": 100 }, "bucket_stats": {}, "state": { "num_processed_objects": 0, "is_finished": true }, "rules_statuses": { "abort-multipart-days-file-prefix": { "state": { "expire": {}, "noncurrent": {}, "is_finished": true }, "rule_process_times": { "process_rule_start_time": 1748251805281, "get_candidates_start_time": 1748251805282, "get_candidates_end_time": 1748251805379, "get_candidates_took_ms": 97, "process_rule_end_time": 1748251805379, "process_rule_took_ms": 98 }, "rule_stats": {} } } }, "bucket123": { "bucket_process_times": { "process_bucket_start_time": 1748251805279, "process_bucket_end_time": 1748251805288, "process_bucket_took_ms": 9 }, "bucket_stats": {}, "state": { "num_processed_objects": 0, "is_finished": true }, "rules_statuses": { "Expiration Rule": { "state": { "expire": { "is_finished": true }, "noncurrent": {}, "is_finished": true }, "rule_process_times": { "process_rule_start_time": 1748251805283, "get_candidates_start_time": 1748251805284, "get_candidates_end_time": 1748251805287, "get_candidates_took_ms": 3, "process_rule_end_time": 1748251805288, "process_rule_took_ms": 5 }, "rule_stats": {} } } } } } } }