From 0548e61910856bfd719e232f89448213e35cb4e6 Mon Sep 17 00:00:00 2001
From: Tayo-pasedaRJ <138471223+Tayo-pasedaRJ@users.noreply.github.com>
Date: Mon, 13 Nov 2023 15:55:52 +0000
Subject: [PATCH] hdfs: added support for list of namenodes in hdfs remote
config
Users can now input a comma separated list of namenodes when writing
config for hdfs remotes.
This is required when you have multiple namenodes in your hdfs cluster
and cannot be certain which namenodes will be in 'standby' or 'active'
states.
This was available before but wasn't documented and didn't use the
correct rclone interfaces.
---
backend/hdfs/fs.go | 2 +-
backend/hdfs/hdfs.go | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/backend/hdfs/fs.go b/backend/hdfs/fs.go
index 8ba9d2366..dfe98bbd4 100644
--- a/backend/hdfs/fs.go
+++ b/backend/hdfs/fs.go
@@ -93,7 +93,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
options := hdfs.ClientOptions{
- Addresses: []string{opt.Namenode},
+ Addresses: opt.Namenode,
UseDatanodeHostname: false,
}
diff --git a/backend/hdfs/hdfs.go b/backend/hdfs/hdfs.go
index 896b7c18f..312aa1e5c 100644
--- a/backend/hdfs/hdfs.go
+++ b/backend/hdfs/hdfs.go
@@ -20,9 +20,10 @@ func init() {
NewFs: NewFs,
Options: []fs.Option{{
Name: "namenode",
- Help: "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
+ Help: "Hadoop name nodes and ports.\n\nE.g. \"namenode-1:8020,namenode-2:8020,...\" to connect to host namenodes at port 8020.",
Required: true,
Sensitive: true,
+ Default: fs.CommaSepList{},
}, {
Name: "username",
Help: "Hadoop user name.",
@@ -65,7 +66,7 @@ and 'privacy'. Used only with KERBEROS enabled.`,
// Options for this backend
type Options struct {
- Namenode string `config:"namenode"`
+ Namenode fs.CommaSepList `config:"namenode"`
Username string `config:"username"`
ServicePrincipalName string `config:"service_principal_name"`
DataTransferProtection string `config:"data_transfer_protection"`