From 8c9e05d1f057af07fe5492ab5910a5932ae3f4ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 15 Apr 2025 08:49:06 +0200 Subject: [PATCH 1/2] Expose more config vars for the posix fs watchers --- services/storage-users/pkg/config/config.go | 11 ++++++----- services/storage-users/pkg/revaconfig/drivers.go | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 7a92de1c16..b33d8ed3fa 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -214,11 +214,12 @@ type PosixDriver struct { EnableFSRevisions bool `yaml:"enable_fs_revisions" env:"STORAGE_USERS_POSIX_ENABLE_FS_REVISIONS" desc:"Allow for generating revisions from changes done to the local storage. Note: This doubles the number of bytes stored on disk because a copy of the current revision is stored to be turned into a revision later." introductionVersion:"1.0.0"` - WatchFS bool `yaml:"watch_fs" env:"STORAGE_USERS_POSIX_WATCH_FS" desc:"Enable the filesystem watcher to detect changes to the filesystem. This is used to detect changes to the filesystem and update the metadata accordingly." introductionVersion:"2.0.0"` - WatchType string `yaml:"watch_type" env:"STORAGE_USERS_POSIX_WATCH_TYPE" desc:"Type of the watcher to use for getting notified about changes to the filesystem. Currently available options are 'inotifywait' (default), 'gpfswatchfolder' and 'gpfsfileauditlogging'." introductionVersion:"1.0.0"` - WatchPath string `yaml:"watch_path" env:"STORAGE_USERS_POSIX_WATCH_PATH" desc:"Path to the watch directory/file. Only applies to the 'gpfsfileauditlogging' and 'inotifywait' watcher, in which case it is the path of the file audit log file/base directory to watch." introductionVersion:"1.0.0"` - WatchFolderKafkaBrokers string `yaml:"watch_folder_kafka_hosts" env:"STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS" desc:"Comma-separated list of kafka brokers to read the watchfolder events from." introductionVersion:"1.0.0"` - InotifyStatsFrequency time.Duration `yaml:"inotify_stats_frequency" env:"STORAGE_USERS_POSIX_INOTIFY_STATS_FREQUENCY" desc:"Frequency to log inotify stats." introductionVersion:"%%NEXT%%"` + WatchFS bool `yaml:"watch_fs" env:"STORAGE_USERS_POSIX_WATCH_FS" desc:"Enable the filesystem watcher to detect changes to the filesystem. This is used to detect changes to the filesystem and update the metadata accordingly." introductionVersion:"2.0.0"` + WatchType string `yaml:"watch_type" env:"STORAGE_USERS_POSIX_WATCH_TYPE" desc:"Type of the watcher to use for getting notified about changes to the filesystem. Currently available options are 'inotifywait' (default), 'cephfs', 'gpfswatchfolder' and 'gpfsfileauditlogging'." introductionVersion:"1.0.0"` + WatchPath string `yaml:"watch_path" env:"STORAGE_USERS_POSIX_WATCH_PATH" desc:"Path to the watch directory/file. Only applies to the 'gpfsfileauditlogging' and 'inotifywait' watcher, in which case it is the path of the file audit log file/base directory to watch." introductionVersion:"1.0.0"` + WatchNotificationBrokers string `yaml:"watch_notification_brokers" env:"STORAGE_USERS_POSIX_WATCH_NOTIFICATION_BROKERS,STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS" desc:"Comma-separated list of kafka brokers to read the watchfolder events from." introductionVersion:"1.0.0"` + WatchRoot string `yaml:"watch_root" env:"STORAGE_USERS_POSIX_WATCH_ROOT" desc:"Path to the watch root directory. Event paths will be considered relative to this path. Only applies to the 'gpswatchfolder' and 'cephfs' watchers." introductionVersion:"%%NEXT%%"` + InotifyStatsFrequency time.Duration `yaml:"inotify_stats_frequency" env:"STORAGE_USERS_POSIX_INOTIFY_STATS_FREQUENCY" desc:"Frequency to log inotify stats." introductionVersion:"%%NEXT%%"` } // Events combines the configuration options for the event bus. diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 6f804f35dd..5d7e460413 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -140,7 +140,8 @@ func Posix(cfg *config.Config, enableFSScan, enableFSWatch bool) map[string]inte "watch_fs": enableFSWatch, "watch_type": cfg.Drivers.Posix.WatchType, "watch_path": cfg.Drivers.Posix.WatchPath, - "watch_folder_kafka_brokers": cfg.Drivers.Posix.WatchFolderKafkaBrokers, + "watch_notification_brokers": cfg.Drivers.Posix.WatchNotificationBrokers, + "watch_root": cfg.Drivers.Posix.WatchRoot, "inotify_stats_frequency": cfg.Drivers.Posix.InotifyStatsFrequency, } } From f8c89b1c3d07e10edc305fe5a95a2f228ca5fcee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 16 Apr 2025 11:01:01 +0200 Subject: [PATCH 2/2] Add deprecation notice for STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS --- services/storage-users/pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index b33d8ed3fa..570045f745 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -217,7 +217,7 @@ type PosixDriver struct { WatchFS bool `yaml:"watch_fs" env:"STORAGE_USERS_POSIX_WATCH_FS" desc:"Enable the filesystem watcher to detect changes to the filesystem. This is used to detect changes to the filesystem and update the metadata accordingly." introductionVersion:"2.0.0"` WatchType string `yaml:"watch_type" env:"STORAGE_USERS_POSIX_WATCH_TYPE" desc:"Type of the watcher to use for getting notified about changes to the filesystem. Currently available options are 'inotifywait' (default), 'cephfs', 'gpfswatchfolder' and 'gpfsfileauditlogging'." introductionVersion:"1.0.0"` WatchPath string `yaml:"watch_path" env:"STORAGE_USERS_POSIX_WATCH_PATH" desc:"Path to the watch directory/file. Only applies to the 'gpfsfileauditlogging' and 'inotifywait' watcher, in which case it is the path of the file audit log file/base directory to watch." introductionVersion:"1.0.0"` - WatchNotificationBrokers string `yaml:"watch_notification_brokers" env:"STORAGE_USERS_POSIX_WATCH_NOTIFICATION_BROKERS,STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS" desc:"Comma-separated list of kafka brokers to read the watchfolder events from." introductionVersion:"1.0.0"` + WatchNotificationBrokers string `yaml:"watch_notification_brokers" env:"STORAGE_USERS_POSIX_WATCH_NOTIFICATION_BROKERS,STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS" desc:"Comma-separated list of kafka brokers to read the watchfolder events from." introductionVersion:"1.0.0" deprecationVersion:"%%NEXT%%" deprecationNotice:"STORAGE_USERS_POSIX_WATCH_FOLDER_KAFKA_BROKERS is deprecated and will be removed in a future version. Please use STORAGE_USERS_POSIX_WATCH_NOTIFICATION_BROKERS instead."` WatchRoot string `yaml:"watch_root" env:"STORAGE_USERS_POSIX_WATCH_ROOT" desc:"Path to the watch root directory. Event paths will be considered relative to this path. Only applies to the 'gpswatchfolder' and 'cephfs' watchers." introductionVersion:"%%NEXT%%"` InotifyStatsFrequency time.Duration `yaml:"inotify_stats_frequency" env:"STORAGE_USERS_POSIX_INOTIFY_STATS_FREQUENCY" desc:"Frequency to log inotify stats." introductionVersion:"%%NEXT%%"` }