diff --git a/backend/azureblob/auth/auth.go b/backend/azureblob/auth/auth.go new file mode 100644 index 000000000..b1e365dec --- /dev/null +++ b/backend/azureblob/auth/auth.go @@ -0,0 +1,584 @@ +// Package auth supplies the authentication and client creation for the azure SDK +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/lib/env" +) + +const ( + // Default storage account, key and blob endpoint for emulator support, + // though it is a base64 key checked in here, it is publicly available secret. + emulatorAccount = "devstoreaccount1" + emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1" +) + +// ConfigOptions is the common authentication options for azure +var ConfigOptions = []fs.Option{{ + Name: "account", + Help: `Azure Storage Account Name. + +Set this to the Azure Storage Account Name in use. + +Leave blank to use SAS URL or Emulator, otherwise it needs to be set. + +If this is blank and if env_auth is set it will be read from the +environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible. +`, + Sensitive: true, +}, { + Name: "env_auth", + Help: `Read credentials from runtime (environment variables, CLI or MSI). + +See the [authentication docs](/azureblob#authentication) for full info.`, + Default: false, +}, { + Name: "key", + Help: `Storage Account Shared Key. + +Leave blank to use SAS URL or Emulator.`, + Sensitive: true, +}, { + Name: "sas_url", + Help: `SAS URL for container level access only. + +Leave blank if using account/key or Emulator.`, + Sensitive: true, +}, { + Name: "connection_string", + Help: `Storage Connection String. + +Connection string for the storage. Leave blank if using other auth methods. +`, + Sensitive: true, +}, { + Name: "tenant", + Help: `ID of the service principal's tenant. Also called its directory ID. + +Set this if using +- Service principal with client secret +- Service principal with certificate +- User with username and password +`, + Sensitive: true, +}, { + Name: "client_id", + Help: `The ID of the client in use. + +Set this if using +- Service principal with client secret +- Service principal with certificate +- User with username and password +`, + Sensitive: true, +}, { + Name: "client_secret", + Help: `One of the service principal's client secrets + +Set this if using +- Service principal with client secret +`, + Sensitive: true, +}, { + Name: "client_certificate_path", + Help: `Path to a PEM or PKCS12 certificate file including the private key. + +Set this if using +- Service principal with certificate +`, +}, { + Name: "client_certificate_password", + Help: `Password for the certificate file (optional). + +Optionally set this if using +- Service principal with certificate + +And the certificate has a password. +`, + IsPassword: true, +}, { + Name: "client_send_certificate_chain", + Help: `Send the certificate chain when using certificate auth. + +Specifies whether an authentication request will include an x5c header +to support subject name / issuer based authentication. When set to +true, authentication requests include the x5c header. + +Optionally set this if using +- Service principal with certificate +`, + Default: false, + Advanced: true, +}, { + Name: "username", + Help: `User name (usually an email address) + +Set this if using +- User with username and password +`, + Advanced: true, + Sensitive: true, +}, { + Name: "password", + Help: `The user's password + +Set this if using +- User with username and password +`, + IsPassword: true, + Advanced: true, +}, { + Name: "service_principal_file", + Help: `Path to file containing credentials for use with a service principal. + +Leave blank normally. Needed only if you want to use a service principal instead of interactive login. + + $ az ad sp create-for-rbac --name "" \ + --role "Storage Blob Data Owner" \ + --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ + > azure-principal.json + +See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details. + +It may be more convenient to put the credentials directly into the +rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + ` +keys instead of setting ` + "`service_principal_file`" + `. +`, + Advanced: true, +}, { + Name: "disable_instance_discovery", + Help: `Skip requesting Microsoft Entra instance metadata + +This should be set true only by applications authenticating in +disconnected clouds, or private clouds such as Azure Stack. + +It determines whether rclone requests Microsoft Entra instance +metadata from ` + "`https://login.microsoft.com/`" + ` before +authenticating. + +Setting this to true will skip this request, making you responsible +for ensuring the configured authority is valid and trustworthy. +`, + Default: false, + Advanced: true, +}, { + Name: "use_msi", + Help: `Use a managed service identity to authenticate (only works in Azure). + +When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/) +to authenticate to Azure Storage instead of a SAS token or account key. + +If the VM(SS) on which this program is running has a system-assigned identity, it will +be used by default. If the resource has no system-assigned but exactly one user-assigned identity, +the user-assigned identity will be used by default. If the resource has multiple user-assigned +identities, the identity to use must be explicitly specified using exactly one of the msi_object_id, +msi_client_id, or msi_mi_res_id parameters.`, + Default: false, + Advanced: true, +}, { + Name: "msi_object_id", + Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.", + Advanced: true, + Sensitive: true, +}, { + Name: "msi_client_id", + Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.", + Advanced: true, + Sensitive: true, +}, { + Name: "msi_mi_res_id", + Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.", + Advanced: true, + Sensitive: true, +}, { + Name: "use_emulator", + Help: "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.", + Default: false, + Advanced: true, +}, { + Name: "use_az", + Help: `Use Azure CLI tool az for authentication + +Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/) +as the sole means of authentication. + +Setting this can be useful if you wish to use the az CLI on a host with +a System Managed Identity that you do not want to use. + +Don't set env_auth at the same time. +`, + Default: false, + Advanced: true, +}, { + Name: "endpoint", + Help: "Endpoint for the service.\n\nLeave blank normally.", + Advanced: true, +}} + +// Options defines the common auth configuration for azure backends +type Options struct { + Account string `config:"account"` + EnvAuth bool `config:"env_auth"` + Key string `config:"key"` + SASURL string `config:"sas_url"` + ConnectionString string `config:"connection_string"` + Tenant string `config:"tenant"` + ClientID string `config:"client_id"` + ClientSecret string `config:"client_secret"` + ClientCertificatePath string `config:"client_certificate_path"` + ClientCertificatePassword string `config:"client_certificate_password"` + ClientSendCertificateChain bool `config:"client_send_certificate_chain"` + Username string `config:"username"` + Password string `config:"password"` + ServicePrincipalFile string `config:"service_principal_file"` + DisableInstanceDiscovery bool `config:"disable_instance_discovery"` + UseMSI bool `config:"use_msi"` + MSIObjectID string `config:"msi_object_id"` + MSIClientID string `config:"msi_client_id"` + MSIResourceID string `config:"msi_mi_res_id"` + UseEmulator bool `config:"use_emulator"` + UseAZ bool `config:"use_az"` + Endpoint string `config:"endpoint"` +} + +type servicePrincipalCredentials struct { + AppID string `json:"appId"` + Password string `json:"password"` + Tenant string `json:"tenant"` +} + +// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli. +func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) { + var spCredentials servicePrincipalCredentials + if err := json.Unmarshal(credentialsData, &spCredentials); err != nil { + return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err) + } + // TODO: support certificate credentials + // Validate all fields present + if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" { + return nil, fmt.Errorf("missing fields in credentials file") + } + return &spCredentials, nil +} + +// Wrap the http.Transport to satisfy the Transporter interface +type transporter struct { + http.RoundTripper +} + +// Make a new transporter +func newTransporter(ctx context.Context) transporter { + return transporter{ + RoundTripper: fshttp.NewTransport(ctx), + } +} + +// Do sends the HTTP request and returns the HTTP response or error. +func (tr transporter) Do(req *http.Request) (*http.Response, error) { + return tr.RoundTripper.RoundTrip(req) +} + +// NewClientOpts should be passed to configure NewClient +type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct { + DefaultBaseURL string // Base URL, eg blob.core.windows.net + Blob bool // set if this is blob storage + RootContainer string // Container that rclone is looking at + NewClient func(serviceURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) + NewClientFromConnectionString func(connectionString string, options *ClientOptions) (*Client, error) + NewClientWithNoCredential func(serviceURL string, options *ClientOptions) (*Client, error) + NewClientWithSharedKeyCredential func(serviceURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) + NewSharedKeyCredential func(accountName, accountKey string) (*SharedKeyCredential, error) + SetClientOptions func(options *ClientOptions, policyClientOptions policy.ClientOptions) +} + +// NewClientResult is returned from NewClient +type NewClientResult[Client any] struct { + Client *Client // Client to access the Service + Cred azcore.TokenCredential // how to generate tokens (may be nil) + UsingSharedKeyCred bool // set if using shared key credentials + Anonymous bool // true if anonymous authentication was used + Container string // Container that SAS URL points to +} + +// NewClient creates a service client from the rclone options +func NewClient[Client, ClientOptions, SharedKeyCredential any](ctx context.Context, conf NewClientOpts[Client, ClientOptions, SharedKeyCredential], opt *Options) (r NewClientResult[Client], err error) { + var sharedKeyCred *SharedKeyCredential + + // Client options specifying our own transport + policyClientOptions := policy.ClientOptions{ + Transport: newTransporter(ctx), + } + // Can't do this with generics (yet) + // clientOpt := service.ClientOptions{ + // ClientOptions: policyClientOptions, + // } + // So call back to user + var clientOpt ClientOptions + conf.SetClientOptions(&clientOpt, policyClientOptions) + + // Here we auth by setting one of cred, sharedKeyCred, client or anonymous + switch { + case opt.EnvAuth: + // Read account from environment if needed + if opt.Account == "" { + opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + } + // Read credentials from the environment + options := azidentity.DefaultAzureCredentialOptions{ + ClientOptions: policyClientOptions, + DisableInstanceDiscovery: opt.DisableInstanceDiscovery, + } + r.Cred, err = azidentity.NewDefaultAzureCredential(&options) + if err != nil { + return r, fmt.Errorf("create azure environment credential failed: %w", err) + } + case opt.UseEmulator: + if opt.Account == "" { + opt.Account = emulatorAccount + } + if opt.Key == "" { + opt.Key = emulatorAccountKey + } + if opt.Endpoint == "" { + opt.Endpoint = emulatorBlobEndpoint + } + if conf.NewSharedKeyCredential == nil { + return r, errors.New("emulator use not supported") + } + sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key) + if err != nil { + return r, fmt.Errorf("create new shared key credential for emulator failed: %w", err) + } + case opt.Account != "" && opt.Key != "": + if conf.NewSharedKeyCredential == nil { + return r, errors.New("shared key credentials not supported") + } + sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key) + if err != nil { + return r, fmt.Errorf("create new shared key credential failed: %w", err) + } + case opt.SASURL != "": + parts, err := sas.ParseURL(opt.SASURL) + if err != nil { + return r, fmt.Errorf("failed to parse SAS URL: %w", err) + } + endpoint := opt.SASURL + r.Container = parts.ContainerName + // Check if we have container level SAS or account level SAS + if conf.Blob && r.Container != "" { + // Container level SAS + if conf.RootContainer != "" && r.Container != conf.RootContainer { + return r, fmt.Errorf("container name in SAS URL (%q) and container provided in command (%q) do not match", r.Container, conf.RootContainer) + } + // Rewrite the endpoint string to be without the container + parts.ContainerName = "" + endpoint = parts.String() + } + r.Client, err = conf.NewClientWithNoCredential(endpoint, &clientOpt) + if err != nil { + return r, fmt.Errorf("unable to create SAS URL client: %w", err) + } + case opt.ConnectionString != "": + r.Client, err = conf.NewClientFromConnectionString(opt.ConnectionString, &clientOpt) + if err != nil { + return r, fmt.Errorf("unable to create connection string client: %w", err) + } + case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "": + // Service principal with client secret + options := azidentity.ClientSecretCredentialOptions{ + ClientOptions: policyClientOptions, + } + r.Cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options) + if err != nil { + return r, fmt.Errorf("error creating a client secret credential: %w", err) + } + case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "": + // Service principal with certificate + // + // Read the certificate + data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath)) + if err != nil { + return r, fmt.Errorf("error reading client certificate file: %w", err) + } + // NewClientCertificateCredential requires at least one *x509.Certificate, and a + // crypto.PrivateKey. + // + // ParseCertificates returns these given certificate data in PEM or PKCS12 format. + // It handles common scenarios but has limitations, for example it doesn't load PEM + // encrypted private keys. + var password []byte + if opt.ClientCertificatePassword != "" { + pw, err := obscure.Reveal(opt.Password) + if err != nil { + return r, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err) + } + password = []byte(pw) + } + certs, key, err := azidentity.ParseCertificates(data, password) + if err != nil { + return r, fmt.Errorf("failed to parse client certificate file: %w", err) + } + options := azidentity.ClientCertificateCredentialOptions{ + ClientOptions: policyClientOptions, + SendCertificateChain: opt.ClientSendCertificateChain, + } + r.Cred, err = azidentity.NewClientCertificateCredential( + opt.Tenant, opt.ClientID, certs, key, &options, + ) + if err != nil { + return r, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err) + } + case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "": + // User with username and password + //nolint:staticcheck // this is deprecated due to Azure policy + options := azidentity.UsernamePasswordCredentialOptions{ + ClientOptions: policyClientOptions, + } + password, err := obscure.Reveal(opt.Password) + if err != nil { + return r, fmt.Errorf("user password decode failed - did you obscure it?: %w", err) + } + r.Cred, err = azidentity.NewUsernamePasswordCredential( + opt.Tenant, opt.ClientID, opt.Username, password, &options, + ) + if err != nil { + return r, fmt.Errorf("authenticate user with password failed: %w", err) + } + case opt.ServicePrincipalFile != "": + // Loading service principal credentials from file. + loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile)) + if err != nil { + return r, fmt.Errorf("error opening service principal credentials file: %w", err) + } + parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds) + if err != nil { + return r, fmt.Errorf("error parsing service principal credentials file: %w", err) + } + options := azidentity.ClientSecretCredentialOptions{ + ClientOptions: policyClientOptions, + } + r.Cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options) + if err != nil { + return r, fmt.Errorf("error creating a client secret credential: %w", err) + } + case opt.UseMSI: + // Specifying a user-assigned identity. Exactly one of the above IDs must be specified. + // Validate and ensure exactly one is set. (To do: better validation.) + var b2i = map[bool]int{false: 0, true: 1} + set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""] + if set > 1 { + return r, errors.New("more than one user-assigned identity ID is set") + } + var options azidentity.ManagedIdentityCredentialOptions + switch { + case opt.MSIClientID != "": + options.ID = azidentity.ClientID(opt.MSIClientID) + case opt.MSIObjectID != "": + // FIXME this doesn't appear to be in the new SDK? + return r, fmt.Errorf("MSI object ID is currently unsupported") + case opt.MSIResourceID != "": + options.ID = azidentity.ResourceID(opt.MSIResourceID) + } + r.Cred, err = azidentity.NewManagedIdentityCredential(&options) + if err != nil { + return r, fmt.Errorf("failed to acquire MSI token: %w", err) + } + case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "": + // Workload Identity based authentication + var options azidentity.ManagedIdentityCredentialOptions + options.ID = azidentity.ClientID(opt.MSIClientID) + + msiCred, err := azidentity.NewManagedIdentityCredential(&options) + if err != nil { + return r, fmt.Errorf("failed to acquire MSI token: %w", err) + } + + getClientAssertions := func(context.Context) (string, error) { + token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"api://AzureADTokenExchange"}, + }) + + if err != nil { + return "", fmt.Errorf("failed to acquire MSI token: %w", err) + } + + return token.Token, nil + } + + assertOpts := &azidentity.ClientAssertionCredentialOptions{} + r.Cred, err = azidentity.NewClientAssertionCredential( + opt.Tenant, + opt.ClientID, + getClientAssertions, + assertOpts) + + if err != nil { + return r, fmt.Errorf("failed to acquire client assertion token: %w", err) + } + case opt.UseAZ: + var options = azidentity.AzureCLICredentialOptions{} + r.Cred, err = azidentity.NewAzureCLICredential(&options) + if err != nil { + return r, fmt.Errorf("failed to create Azure CLI credentials: %w", err) + } + case opt.Account != "": + // Anonymous access + r.Anonymous = true + default: + return r, errors.New("no authentication method configured") + } + + // Make the client if not already created + if r.Client == nil { + // Work out what the endpoint is if it is still unset + if opt.Endpoint == "" { + if opt.Account == "" { + return r, fmt.Errorf("account must be set: can't make service URL") + } + u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, conf.DefaultBaseURL)) + if err != nil { + return r, fmt.Errorf("failed to make azure storage URL from account: %w", err) + } + opt.Endpoint = u.String() + } + if sharedKeyCred != nil { + // Shared key cred + r.Client, err = conf.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt) + if err != nil { + return r, fmt.Errorf("create client with shared key failed: %w", err) + } + r.UsingSharedKeyCred = true + } else if r.Cred != nil { + // Azidentity cred + r.Client, err = conf.NewClient(opt.Endpoint, r.Cred, &clientOpt) + if err != nil { + return r, fmt.Errorf("create client failed: %w", err) + } + } else if r.Anonymous { + // Anonymous public access + r.Client, err = conf.NewClientWithNoCredential(opt.Endpoint, &clientOpt) + if err != nil { + return r, fmt.Errorf("create public client failed: %w", err) + } + } + } + if r.Client == nil { + return r, fmt.Errorf("internal error: auth failed to make credentials or client") + } + return r, nil +} diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 6b72fc11b..52d9fcd38 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -11,13 +11,10 @@ import ( "encoding/base64" "encoding/binary" "encoding/hex" - "encoding/json" "errors" "fmt" "io" "net/http" - "net/url" - "os" "path" "slices" "sort" @@ -28,27 +25,24 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/rclone/rclone/backend/azureblob/auth" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/chunksize" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" - "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" - "github.com/rclone/rclone/lib/env" "github.com/rclone/rclone/lib/multipart" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/pool" @@ -68,12 +62,7 @@ const ( storageDefaultBaseURL = "blob.core.windows.net" defaultChunkSize = 4 * fs.Mebi defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone - // Default storage account, key and blob endpoint for emulator support, - // though it is a base64 key checked in here, it is publicly available secret. - emulatorAccount = "devstoreaccount1" - emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1" - sasCopyValidity = time.Hour // how long SAS should last when doing server side copy + sasCopyValidity = time.Hour // how long SAS should last when doing server side copy ) var ( @@ -136,199 +125,7 @@ func init() { System: systemMetadataInfo, Help: `User metadata is stored as x-ms-meta- keys. Azure metadata keys are case insensitive and are always returned in lower case.`, }, - Options: []fs.Option{{ - Name: "account", - Help: `Azure Storage Account Name. - -Set this to the Azure Storage Account Name in use. - -Leave blank to use SAS URL or Emulator, otherwise it needs to be set. - -If this is blank and if env_auth is set it will be read from the -environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible. -`, - Sensitive: true, - }, { - Name: "env_auth", - Help: `Read credentials from runtime (environment variables, CLI or MSI). - -See the [authentication docs](/azureblob#authentication) for full info.`, - Default: false, - }, { - Name: "key", - Help: `Storage Account Shared Key. - -Leave blank to use SAS URL or Emulator.`, - Sensitive: true, - }, { - Name: "sas_url", - Help: `SAS URL for container level access only. - -Leave blank if using account/key or Emulator.`, - Sensitive: true, - }, { - Name: "tenant", - Help: `ID of the service principal's tenant. Also called its directory ID. - -Set this if using -- Service principal with client secret -- Service principal with certificate -- User with username and password -`, - Sensitive: true, - }, { - Name: "client_id", - Help: `The ID of the client in use. - -Set this if using -- Service principal with client secret -- Service principal with certificate -- User with username and password -`, - Sensitive: true, - }, { - Name: "client_secret", - Help: `One of the service principal's client secrets - -Set this if using -- Service principal with client secret -`, - Sensitive: true, - }, { - Name: "client_certificate_path", - Help: `Path to a PEM or PKCS12 certificate file including the private key. - -Set this if using -- Service principal with certificate -`, - }, { - Name: "client_certificate_password", - Help: `Password for the certificate file (optional). - -Optionally set this if using -- Service principal with certificate - -And the certificate has a password. -`, - IsPassword: true, - }, { - Name: "client_send_certificate_chain", - Help: `Send the certificate chain when using certificate auth. - -Specifies whether an authentication request will include an x5c header -to support subject name / issuer based authentication. When set to -true, authentication requests include the x5c header. - -Optionally set this if using -- Service principal with certificate -`, - Default: false, - Advanced: true, - }, { - Name: "username", - Help: `User name (usually an email address) - -Set this if using -- User with username and password -`, - Advanced: true, - Sensitive: true, - }, { - Name: "password", - Help: `The user's password - -Set this if using -- User with username and password -`, - IsPassword: true, - Advanced: true, - }, { - Name: "service_principal_file", - Help: `Path to file containing credentials for use with a service principal. - -Leave blank normally. Needed only if you want to use a service principal instead of interactive login. - - $ az ad sp create-for-rbac --name "" \ - --role "Storage Blob Data Owner" \ - --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ - > azure-principal.json - -See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details. - -It may be more convenient to put the credentials directly into the -rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + ` -keys instead of setting ` + "`service_principal_file`" + `. -`, - Advanced: true, - }, { - Name: "disable_instance_discovery", - Help: `Skip requesting Microsoft Entra instance metadata - -This should be set true only by applications authenticating in -disconnected clouds, or private clouds such as Azure Stack. - -It determines whether rclone requests Microsoft Entra instance -metadata from ` + "`https://login.microsoft.com/`" + ` before -authenticating. - -Setting this to true will skip this request, making you responsible -for ensuring the configured authority is valid and trustworthy. -`, - Default: false, - Advanced: true, - }, { - Name: "use_msi", - Help: `Use a managed service identity to authenticate (only works in Azure). - -When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/) -to authenticate to Azure Storage instead of a SAS token or account key. - -If the VM(SS) on which this program is running has a system-assigned identity, it will -be used by default. If the resource has no system-assigned but exactly one user-assigned identity, -the user-assigned identity will be used by default. If the resource has multiple user-assigned -identities, the identity to use must be explicitly specified using exactly one of the msi_object_id, -msi_client_id, or msi_mi_res_id parameters.`, - Default: false, - Advanced: true, - }, { - Name: "msi_object_id", - Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "msi_client_id", - Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "msi_mi_res_id", - Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "use_emulator", - Help: "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.", - Default: false, - Advanced: true, - }, { - Name: "use_az", - Help: `Use Azure CLI tool az for authentication - -Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/) -as the sole means of authentication. - -Setting this can be useful if you wish to use the az CLI on a host with -a System Managed Identity that you do not want to use. - -Don't set env_auth at the same time. -`, - Default: false, - Advanced: true, - }, { - Name: "endpoint", - Help: "Endpoint for the service.\n\nLeave blank normally.", - Advanced: true, - }, { + Options: slices.Concat(auth.ConfigOptions, []fs.Option{{ Name: "upload_cutoff", Help: "Cutoff for switching to chunked upload (<= 256 MiB) (deprecated).", Advanced: true, @@ -541,70 +338,50 @@ rclone does if you know the container exists already. Default: "", Exclusive: true, Advanced: true, - }}, + }}), }) } // Options defines the configuration for this backend type Options struct { - Account string `config:"account"` - EnvAuth bool `config:"env_auth"` - Key string `config:"key"` - SASURL string `config:"sas_url"` - Tenant string `config:"tenant"` - ClientID string `config:"client_id"` - ClientSecret string `config:"client_secret"` - ClientCertificatePath string `config:"client_certificate_path"` - ClientCertificatePassword string `config:"client_certificate_password"` - ClientSendCertificateChain bool `config:"client_send_certificate_chain"` - Username string `config:"username"` - Password string `config:"password"` - ServicePrincipalFile string `config:"service_principal_file"` - DisableInstanceDiscovery bool `config:"disable_instance_discovery"` - UseMSI bool `config:"use_msi"` - MSIObjectID string `config:"msi_object_id"` - MSIClientID string `config:"msi_client_id"` - MSIResourceID string `config:"msi_mi_res_id"` - UseAZ bool `config:"use_az"` - Endpoint string `config:"endpoint"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` - CopyConcurrency int `config:"copy_concurrency"` - UseCopyBlob bool `config:"use_copy_blob"` - UploadConcurrency int `config:"upload_concurrency"` - ListChunkSize uint `config:"list_chunk"` - AccessTier string `config:"access_tier"` - ArchiveTierDelete bool `config:"archive_tier_delete"` - UseEmulator bool `config:"use_emulator"` - DisableCheckSum bool `config:"disable_checksum"` - Enc encoder.MultiEncoder `config:"encoding"` - PublicAccess string `config:"public_access"` - DirectoryMarkers bool `config:"directory_markers"` - NoCheckContainer bool `config:"no_check_container"` - NoHeadObject bool `config:"no_head_object"` - DeleteSnapshots string `config:"delete_snapshots"` + auth.Options + ChunkSize fs.SizeSuffix `config:"chunk_size"` + CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` + CopyConcurrency int `config:"copy_concurrency"` + UseCopyBlob bool `config:"use_copy_blob"` + UploadConcurrency int `config:"upload_concurrency"` + ListChunkSize uint `config:"list_chunk"` + AccessTier string `config:"access_tier"` + ArchiveTierDelete bool `config:"archive_tier_delete"` + DisableCheckSum bool `config:"disable_checksum"` + Enc encoder.MultiEncoder `config:"encoding"` + PublicAccess string `config:"public_access"` + DirectoryMarkers bool `config:"directory_markers"` + NoCheckContainer bool `config:"no_check_container"` + NoHeadObject bool `config:"no_head_object"` + DeleteSnapshots string `config:"delete_snapshots"` } // Fs represents a remote azure server type Fs struct { - name string // name of this remote - root string // the path we are working on if any - opt Options // parsed config options - ci *fs.ConfigInfo // global config - features *fs.Features // optional features - cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache - cntSVCcache map[string]*container.Client // reference to containerClient per container - svc *service.Client // client to access azblob - cred azcore.TokenCredential // how to generate tokens (may be nil) - sharedKeyCred *service.SharedKeyCredential // shared key credentials (may be nil) - anonymous bool // if this is anonymous access - rootContainer string // container part of root (if any) - rootDirectory string // directory part of root (if any) - isLimited bool // if limited to one container - cache *bucket.Cache // cache for container creation status - pacer *fs.Pacer // To pace and retry the API calls - uploadToken *pacer.TokenDispenser // control concurrency - publicAccess container.PublicAccessType // Container Public Access Level + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features + cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache + cntSVCcache map[string]*container.Client // reference to containerClient per container + svc *service.Client // client to access azblob + cred azcore.TokenCredential // how to generate tokens (may be nil) + usingSharedKeyCred bool // set if using shared key credentials + anonymous bool // if this is anonymous access + rootContainer string // container part of root (if any) + rootDirectory string // directory part of root (if any) + isLimited bool // if limited to one container + cache *bucket.Cache // cache for container creation status + pacer *fs.Pacer // To pace and retry the API calls + uploadToken *pacer.TokenDispenser // control concurrency + publicAccess container.PublicAccessType // Container Public Access Level // user delegation cache userDelegationMu sync.Mutex @@ -767,49 +544,12 @@ func (f *Fs) setCopyCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { return } -type servicePrincipalCredentials struct { - AppID string `json:"appId"` - Password string `json:"password"` - Tenant string `json:"tenant"` -} - -// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli. -func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) { - var spCredentials servicePrincipalCredentials - if err := json.Unmarshal(credentialsData, &spCredentials); err != nil { - return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err) - } - // TODO: support certificate credentials - // Validate all fields present - if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" { - return nil, fmt.Errorf("missing fields in credentials file") - } - return &spCredentials, nil -} - // setRoot changes the root of the Fs func (f *Fs) setRoot(root string) { f.root = parsePath(root) f.rootContainer, f.rootDirectory = bucket.Split(f.root) } -// Wrap the http.Transport to satisfy the Transporter interface -type transporter struct { - http.RoundTripper -} - -// Make a new transporter -func newTransporter(ctx context.Context) transporter { - return transporter{ - RoundTripper: fshttp.NewTransport(ctx), - } -} - -// Do sends the HTTP request and returns the HTTP response or error. -func (tr transporter) Do(req *http.Request) (*http.Response, error) { - return tr.RoundTripper.RoundTrip(req) -} - // NewFs constructs an Fs from the path, container:path func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct @@ -869,255 +609,32 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e fs.Debugf(f, "Using directory markers") } - // Client options specifying our own transport - policyClientOptions := policy.ClientOptions{ - Transport: newTransporter(ctx), + conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{ + DefaultBaseURL: storageDefaultBaseURL, + RootContainer: f.rootContainer, + Blob: true, + NewClient: service.NewClient, + NewClientFromConnectionString: service.NewClientFromConnectionString, + NewClientWithNoCredential: service.NewClientWithNoCredential, + NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential, + NewSharedKeyCredential: service.NewSharedKeyCredential, + SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) { + options.ClientOptions = policyClientOptions + }, } - clientOpt := service.ClientOptions{ - ClientOptions: policyClientOptions, + res, err := auth.NewClient(ctx, conf, &opt.Options) + if err != nil { + return nil, err } + f.svc = res.Client + f.cred = res.Cred + f.usingSharedKeyCred = res.UsingSharedKeyCred + f.anonymous = res.Anonymous - // Here we auth by setting one of f.cred, f.sharedKeyCred, f.svc or f.anonymous - switch { - case opt.EnvAuth: - // Read account from environment if needed - if opt.Account == "" { - opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") - } - // Read credentials from the environment - options := azidentity.DefaultAzureCredentialOptions{ - ClientOptions: policyClientOptions, - DisableInstanceDiscovery: opt.DisableInstanceDiscovery, - } - f.cred, err = azidentity.NewDefaultAzureCredential(&options) - if err != nil { - return nil, fmt.Errorf("create azure environment credential failed: %w", err) - } - case opt.UseEmulator: - if opt.Account == "" { - opt.Account = emulatorAccount - } - if opt.Key == "" { - opt.Key = emulatorAccountKey - } - if opt.Endpoint == "" { - opt.Endpoint = emulatorBlobEndpoint - } - f.sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key) - if err != nil { - return nil, fmt.Errorf("create new shared key credential for emulator failed: %w", err) - } - case opt.Account != "" && opt.Key != "": - f.sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key) - if err != nil { - return nil, fmt.Errorf("create new shared key credential failed: %w", err) - } - case opt.SASURL != "": - parts, err := sas.ParseURL(opt.SASURL) - if err != nil { - return nil, fmt.Errorf("failed to parse SAS URL: %w", err) - } - endpoint := opt.SASURL - containerName := parts.ContainerName - // Check if we have container level SAS or account level SAS - if containerName != "" { - // Container level SAS - if f.rootContainer != "" && containerName != f.rootContainer { - return nil, fmt.Errorf("container name in SAS URL (%q) and container provided in command (%q) do not match", containerName, f.rootContainer) - } - // Rewrite the endpoint string to be without the container - parts.ContainerName = "" - endpoint = parts.String() - } - f.svc, err = service.NewClientWithNoCredential(endpoint, &clientOpt) - if err != nil { - return nil, fmt.Errorf("unable to create SAS URL client: %w", err) - } - // if using Container level SAS put the container client into the cache - if containerName != "" { - _ = f.cntSVC(containerName) - f.isLimited = true - } - case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "": - // Service principal with client secret - options := azidentity.ClientSecretCredentialOptions{ - ClientOptions: policyClientOptions, - } - f.cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options) - if err != nil { - return nil, fmt.Errorf("error creating a client secret credential: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "": - // Service principal with certificate - // - // Read the certificate - data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath)) - if err != nil { - return nil, fmt.Errorf("error reading client certificate file: %w", err) - } - // NewClientCertificateCredential requires at least one *x509.Certificate, and a - // crypto.PrivateKey. - // - // ParseCertificates returns these given certificate data in PEM or PKCS12 format. - // It handles common scenarios but has limitations, for example it doesn't load PEM - // encrypted private keys. - var password []byte - if opt.ClientCertificatePassword != "" { - pw, err := obscure.Reveal(opt.Password) - if err != nil { - return nil, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err) - } - password = []byte(pw) - } - certs, key, err := azidentity.ParseCertificates(data, password) - if err != nil { - return nil, fmt.Errorf("failed to parse client certificate file: %w", err) - } - options := azidentity.ClientCertificateCredentialOptions{ - ClientOptions: policyClientOptions, - SendCertificateChain: opt.ClientSendCertificateChain, - } - f.cred, err = azidentity.NewClientCertificateCredential( - opt.Tenant, opt.ClientID, certs, key, &options, - ) - if err != nil { - return nil, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "": - // User with username and password - //nolint:staticcheck // this is deprecated due to Azure policy - options := azidentity.UsernamePasswordCredentialOptions{ - ClientOptions: policyClientOptions, - } - password, err := obscure.Reveal(opt.Password) - if err != nil { - return nil, fmt.Errorf("user password decode failed - did you obscure it?: %w", err) - } - f.cred, err = azidentity.NewUsernamePasswordCredential( - opt.Tenant, opt.ClientID, opt.Username, password, &options, - ) - if err != nil { - return nil, fmt.Errorf("authenticate user with password failed: %w", err) - } - case opt.ServicePrincipalFile != "": - // Loading service principal credentials from file. - loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile)) - if err != nil { - return nil, fmt.Errorf("error opening service principal credentials file: %w", err) - } - parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds) - if err != nil { - return nil, fmt.Errorf("error parsing service principal credentials file: %w", err) - } - options := azidentity.ClientSecretCredentialOptions{ - ClientOptions: policyClientOptions, - } - f.cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options) - if err != nil { - return nil, fmt.Errorf("error creating a client secret credential: %w", err) - } - case opt.UseMSI: - // Specifying a user-assigned identity. Exactly one of the above IDs must be specified. - // Validate and ensure exactly one is set. (To do: better validation.) - var b2i = map[bool]int{false: 0, true: 1} - set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""] - if set > 1 { - return nil, errors.New("more than one user-assigned identity ID is set") - } - var options azidentity.ManagedIdentityCredentialOptions - switch { - case opt.MSIClientID != "": - options.ID = azidentity.ClientID(opt.MSIClientID) - case opt.MSIObjectID != "": - // FIXME this doesn't appear to be in the new SDK? - return nil, fmt.Errorf("MSI object ID is currently unsupported") - case opt.MSIResourceID != "": - options.ID = azidentity.ResourceID(opt.MSIResourceID) - } - f.cred, err = azidentity.NewManagedIdentityCredential(&options) - if err != nil { - return nil, fmt.Errorf("failed to acquire MSI token: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "": - // Workload Identity based authentication - var options azidentity.ManagedIdentityCredentialOptions - options.ID = azidentity.ClientID(opt.MSIClientID) - - msiCred, err := azidentity.NewManagedIdentityCredential(&options) - if err != nil { - return nil, fmt.Errorf("failed to acquire MSI token: %w", err) - } - - getClientAssertions := func(context.Context) (string, error) { - token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ - Scopes: []string{"api://AzureADTokenExchange"}, - }) - - if err != nil { - return "", fmt.Errorf("failed to acquire MSI token: %w", err) - } - - return token.Token, nil - } - - assertOpts := &azidentity.ClientAssertionCredentialOptions{} - f.cred, err = azidentity.NewClientAssertionCredential( - opt.Tenant, - opt.ClientID, - getClientAssertions, - assertOpts) - - if err != nil { - return nil, fmt.Errorf("failed to acquire client assertion token: %w", err) - } - case opt.UseAZ: - var options = azidentity.AzureCLICredentialOptions{} - f.cred, err = azidentity.NewAzureCLICredential(&options) - if err != nil { - return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err) - } - case opt.Account != "": - // Anonymous access - f.anonymous = true - default: - return nil, errors.New("no authentication method configured") - } - - // Make the client if not already created - if f.svc == nil { - // Work out what the endpoint is if it is still unset - if opt.Endpoint == "" { - if opt.Account == "" { - return nil, fmt.Errorf("account must be set: can't make service URL") - } - u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL)) - if err != nil { - return nil, fmt.Errorf("failed to make azure storage URL from account: %w", err) - } - opt.Endpoint = u.String() - } - if f.sharedKeyCred != nil { - // Shared key cred - f.svc, err = service.NewClientWithSharedKeyCredential(opt.Endpoint, f.sharedKeyCred, &clientOpt) - if err != nil { - return nil, fmt.Errorf("create client with shared key failed: %w", err) - } - } else if f.cred != nil { - // Azidentity cred - f.svc, err = service.NewClient(opt.Endpoint, f.cred, &clientOpt) - if err != nil { - return nil, fmt.Errorf("create client failed: %w", err) - } - } else if f.anonymous { - // Anonymous public access - f.svc, err = service.NewClientWithNoCredential(opt.Endpoint, &clientOpt) - if err != nil { - return nil, fmt.Errorf("create public client failed: %w", err) - } - } - } - if f.svc == nil { - return nil, fmt.Errorf("internal error: auth failed to make credentials or client") + // if using Container level SAS put the container client into the cache + if opt.SASURL != "" && res.Container != "" { + _ = f.cntSVC(res.Container) + f.isLimited = true } if f.rootContainer != "" && f.rootDirectory != "" { @@ -2169,7 +1686,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e // Append the SAS to the URL srcURL = srcBlobSVC.URL() + "?" + queryParameters.Encode() - case f.sharedKeyCred != nil: + case f.usingSharedKeyCred: // Generate a short lived SAS URL if using shared key credentials expiry := time.Now().Add(sasCopyValidity) sasOptions := blob.GetSASURLOptions{} diff --git a/backend/azurefiles/azurefiles.go b/backend/azurefiles/azurefiles.go index d5a6a66a4..271e27ca4 100644 --- a/backend/azurefiles/azurefiles.go +++ b/backend/azurefiles/azurefiles.go @@ -29,36 +29,28 @@ import ( "context" "crypto/md5" "encoding/hex" - "encoding/json" - "errors" "fmt" "io" - "net/http" - "net/url" - "os" "path" + "slices" "strings" "sync" "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/share" + "github.com/rclone/rclone/backend/azureblob/auth" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/lib/encoder" - "github.com/rclone/rclone/lib/env" "github.com/rclone/rclone/lib/readers" ) @@ -73,199 +65,12 @@ func init() { Name: "azurefiles", Description: "Microsoft Azure Files", NewFs: NewFs, - Options: []fs.Option{{ - Name: "account", - Help: `Azure Storage Account Name. - -Set this to the Azure Storage Account Name in use. - -Leave blank to use SAS URL or connection string, otherwise it needs to be set. - -If this is blank and if env_auth is set it will be read from the -environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible. -`, - Sensitive: true, - }, { + Options: slices.Concat(auth.ConfigOptions, []fs.Option{{ Name: "share_name", Help: `Azure Files Share Name. This is required and is the name of the share to access. `, - }, { - Name: "env_auth", - Help: `Read credentials from runtime (environment variables, CLI or MSI). - -See the [authentication docs](/azurefiles#authentication) for full info.`, - Default: false, - }, { - Name: "key", - Help: `Storage Account Shared Key. - -Leave blank to use SAS URL or connection string.`, - Sensitive: true, - }, { - Name: "sas_url", - Help: `SAS URL. - -Leave blank if using account/key or connection string.`, - Sensitive: true, - }, { - Name: "connection_string", - Help: `Azure Files Connection String.`, - Sensitive: true, - }, { - Name: "tenant", - Help: `ID of the service principal's tenant. Also called its directory ID. - -Set this if using -- Service principal with client secret -- Service principal with certificate -- User with username and password -`, - Sensitive: true, - }, { - Name: "client_id", - Help: `The ID of the client in use. - -Set this if using -- Service principal with client secret -- Service principal with certificate -- User with username and password -`, - Sensitive: true, - }, { - Name: "client_secret", - Help: `One of the service principal's client secrets - -Set this if using -- Service principal with client secret -`, - Sensitive: true, - }, { - Name: "client_certificate_path", - Help: `Path to a PEM or PKCS12 certificate file including the private key. - -Set this if using -- Service principal with certificate -`, - }, { - Name: "client_certificate_password", - Help: `Password for the certificate file (optional). - -Optionally set this if using -- Service principal with certificate - -And the certificate has a password. -`, - IsPassword: true, - }, { - Name: "client_send_certificate_chain", - Help: `Send the certificate chain when using certificate auth. - -Specifies whether an authentication request will include an x5c header -to support subject name / issuer based authentication. When set to -true, authentication requests include the x5c header. - -Optionally set this if using -- Service principal with certificate -`, - Default: false, - Advanced: true, - }, { - Name: "username", - Help: `User name (usually an email address) - -Set this if using -- User with username and password -`, - Advanced: true, - Sensitive: true, - }, { - Name: "password", - Help: `The user's password - -Set this if using -- User with username and password -`, - IsPassword: true, - Advanced: true, - }, { - Name: "service_principal_file", - Help: `Path to file containing credentials for use with a service principal. - -Leave blank normally. Needed only if you want to use a service principal instead of interactive login. - - $ az ad sp create-for-rbac --name "" \ - --role "Storage Files Data Owner" \ - --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ - > azure-principal.json - -See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to files data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details. - -**NB** this section needs updating for Azure Files - pull requests appreciated! - -It may be more convenient to put the credentials directly into the -rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + ` -keys instead of setting ` + "`service_principal_file`" + `. -`, - Advanced: true, - }, { - Name: "use_msi", - Help: `Use a managed service identity to authenticate (only works in Azure). - -When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/) -to authenticate to Azure Storage instead of a SAS token or account key. - -If the VM(SS) on which this program is running has a system-assigned identity, it will -be used by default. If the resource has no system-assigned but exactly one user-assigned identity, -the user-assigned identity will be used by default. If the resource has multiple user-assigned -identities, the identity to use must be explicitly specified using exactly one of the msi_object_id, -msi_client_id, or msi_mi_res_id parameters.`, - Default: false, - Advanced: true, - }, { - Name: "msi_object_id", - Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "msi_client_id", - Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "msi_mi_res_id", - Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.", - Advanced: true, - Sensitive: true, - }, { - Name: "disable_instance_discovery", - Help: `Skip requesting Microsoft Entra instance metadata -This should be set true only by applications authenticating in -disconnected clouds, or private clouds such as Azure Stack. -It determines whether rclone requests Microsoft Entra instance -metadata from ` + "`https://login.microsoft.com/`" + ` before -authenticating. -Setting this to true will skip this request, making you responsible -for ensuring the configured authority is valid and trustworthy. -`, - Default: false, - Advanced: true, - }, { - Name: "use_az", - Help: `Use Azure CLI tool az for authentication -Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/) -as the sole means of authentication. -Setting this can be useful if you wish to use the az CLI on a host with -a System Managed Identity that you do not want to use. -Don't set env_auth at the same time. -`, - Default: false, - Advanced: true, - }, { - Name: "endpoint", - Help: "Endpoint for the service.\n\nLeave blank normally.", - Advanced: true, }, { Name: "chunk_size", Help: `Upload chunk size. @@ -323,38 +128,18 @@ You will need this much free space in the share as the file will be this size te encoder.EncodeInvalidUtf8 | encoder.EncodeCtl | encoder.EncodeDel | encoder.EncodeDot | encoder.EncodeRightPeriod), - }}, + }}), }) } // Options defines the configuration for this backend type Options struct { - Account string `config:"account"` - ShareName string `config:"share_name"` - EnvAuth bool `config:"env_auth"` - Key string `config:"key"` - SASURL string `config:"sas_url"` - ConnectionString string `config:"connection_string"` - Tenant string `config:"tenant"` - ClientID string `config:"client_id"` - ClientSecret string `config:"client_secret"` - ClientCertificatePath string `config:"client_certificate_path"` - ClientCertificatePassword string `config:"client_certificate_password"` - ClientSendCertificateChain bool `config:"client_send_certificate_chain"` - Username string `config:"username"` - Password string `config:"password"` - ServicePrincipalFile string `config:"service_principal_file"` - DisableInstanceDiscovery bool `config:"disable_instance_discovery"` - UseMSI bool `config:"use_msi"` - MSIObjectID string `config:"msi_object_id"` - MSIClientID string `config:"msi_client_id"` - MSIResourceID string `config:"msi_mi_res_id"` - UseAZ bool `config:"use_az"` - Endpoint string `config:"endpoint"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - MaxStreamSize fs.SizeSuffix `config:"max_stream_size"` - UploadConcurrency int `config:"upload_concurrency"` - Enc encoder.MultiEncoder `config:"encoding"` + auth.Options + ShareName string `config:"share_name"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + MaxStreamSize fs.SizeSuffix `config:"max_stream_size"` + UploadConcurrency int `config:"upload_concurrency"` + Enc encoder.MultiEncoder `config:"encoding"` } // Fs represents a root directory inside a share. The root directory can be "" @@ -377,266 +162,29 @@ type Object struct { contentType string // content type if known } -// Wrap the http.Transport to satisfy the Transporter interface -type transporter struct { - http.RoundTripper -} - -// Make a new transporter -func newTransporter(ctx context.Context) transporter { - return transporter{ - RoundTripper: fshttp.NewTransport(ctx), - } -} - -// Do sends the HTTP request and returns the HTTP response or error. -func (tr transporter) Do(req *http.Request) (*http.Response, error) { - return tr.RoundTripper.RoundTrip(req) -} - -type servicePrincipalCredentials struct { - AppID string `json:"appId"` - Password string `json:"password"` - Tenant string `json:"tenant"` -} - -// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli. -func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) { - var spCredentials servicePrincipalCredentials - if err := json.Unmarshal(credentialsData, &spCredentials); err != nil { - return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err) - } - // TODO: support certificate credentials - // Validate all fields present - if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" { - return nil, fmt.Errorf("missing fields in credentials file") - } - return &spCredentials, nil -} - // Factored out from NewFs so that it can be tested with opt *Options and without m configmap.Mapper func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.Fs, error) { - // Client options specifying our own transport - policyClientOptions := policy.ClientOptions{ - Transport: newTransporter(ctx), + conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{ + DefaultBaseURL: storageDefaultBaseURL, + NewClient: service.NewClient, + NewClientFromConnectionString: service.NewClientFromConnectionString, + NewClientWithNoCredential: service.NewClientWithNoCredential, + NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential, + NewSharedKeyCredential: service.NewSharedKeyCredential, + SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) { + options.ClientOptions = policyClientOptions + }, } - backup := service.ShareTokenIntentBackup - clientOpt := service.ClientOptions{ - ClientOptions: policyClientOptions, - FileRequestIntent: &backup, + res, err := auth.NewClient(ctx, conf, &opt.Options) + if err != nil { + return nil, err } + // f.svc = res.Client + // f.cred = res.Cred + // f.sharedKeyCred = res.SharedKeyCred + // f.anonymous = res.Anonymous - // Here we auth by setting one of cred, sharedKeyCred or f.client - var ( - cred azcore.TokenCredential - sharedKeyCred *service.SharedKeyCredential - client *service.Client - err error - ) - switch { - case opt.EnvAuth: - // Read account from environment if needed - if opt.Account == "" { - opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") - } - // Read credentials from the environment - options := azidentity.DefaultAzureCredentialOptions{ - ClientOptions: policyClientOptions, - DisableInstanceDiscovery: opt.DisableInstanceDiscovery, - } - cred, err = azidentity.NewDefaultAzureCredential(&options) - if err != nil { - return nil, fmt.Errorf("create azure environment credential failed: %w", err) - } - case opt.Account != "" && opt.Key != "": - sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key) - if err != nil { - return nil, fmt.Errorf("create new shared key credential failed: %w", err) - } - case opt.UseAZ: - options := azidentity.AzureCLICredentialOptions{} - cred, err = azidentity.NewAzureCLICredential(&options) - fmt.Println(cred) - if err != nil { - return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err) - } - case opt.SASURL != "": - client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt) - if err != nil { - return nil, fmt.Errorf("unable to create SAS URL client: %w", err) - } - case opt.ConnectionString != "": - client, err = service.NewClientFromConnectionString(opt.ConnectionString, &clientOpt) - if err != nil { - return nil, fmt.Errorf("unable to create connection string client: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "": - // Service principal with client secret - options := azidentity.ClientSecretCredentialOptions{ - ClientOptions: policyClientOptions, - } - cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options) - if err != nil { - return nil, fmt.Errorf("error creating a client secret credential: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "": - // Service principal with certificate - // - // Read the certificate - data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath)) - if err != nil { - return nil, fmt.Errorf("error reading client certificate file: %w", err) - } - // NewClientCertificateCredential requires at least one *x509.Certificate, and a - // crypto.PrivateKey. - // - // ParseCertificates returns these given certificate data in PEM or PKCS12 format. - // It handles common scenarios but has limitations, for example it doesn't load PEM - // encrypted private keys. - var password []byte - if opt.ClientCertificatePassword != "" { - pw, err := obscure.Reveal(opt.Password) - if err != nil { - return nil, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err) - } - password = []byte(pw) - } - certs, key, err := azidentity.ParseCertificates(data, password) - if err != nil { - return nil, fmt.Errorf("failed to parse client certificate file: %w", err) - } - options := azidentity.ClientCertificateCredentialOptions{ - ClientOptions: policyClientOptions, - SendCertificateChain: opt.ClientSendCertificateChain, - } - cred, err = azidentity.NewClientCertificateCredential( - opt.Tenant, opt.ClientID, certs, key, &options, - ) - if err != nil { - return nil, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "": - // User with username and password - //nolint:staticcheck // this is deprecated due to Azure policy - options := azidentity.UsernamePasswordCredentialOptions{ - ClientOptions: policyClientOptions, - } - password, err := obscure.Reveal(opt.Password) - if err != nil { - return nil, fmt.Errorf("user password decode failed - did you obscure it?: %w", err) - } - cred, err = azidentity.NewUsernamePasswordCredential( - opt.Tenant, opt.ClientID, opt.Username, password, &options, - ) - if err != nil { - return nil, fmt.Errorf("authenticate user with password failed: %w", err) - } - case opt.ServicePrincipalFile != "": - // Loading service principal credentials from file. - loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile)) - if err != nil { - return nil, fmt.Errorf("error opening service principal credentials file: %w", err) - } - parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds) - if err != nil { - return nil, fmt.Errorf("error parsing service principal credentials file: %w", err) - } - options := azidentity.ClientSecretCredentialOptions{ - ClientOptions: policyClientOptions, - } - cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options) - if err != nil { - return nil, fmt.Errorf("error creating a client secret credential: %w", err) - } - case opt.UseMSI: - // Specifying a user-assigned identity. Exactly one of the above IDs must be specified. - // Validate and ensure exactly one is set. (To do: better validation.) - b2i := map[bool]int{false: 0, true: 1} - set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""] - if set > 1 { - return nil, errors.New("more than one user-assigned identity ID is set") - } - var options azidentity.ManagedIdentityCredentialOptions - switch { - case opt.MSIClientID != "": - options.ID = azidentity.ClientID(opt.MSIClientID) - case opt.MSIObjectID != "": - // FIXME this doesn't appear to be in the new SDK? - return nil, fmt.Errorf("MSI object ID is currently unsupported") - case opt.MSIResourceID != "": - options.ID = azidentity.ResourceID(opt.MSIResourceID) - } - cred, err = azidentity.NewManagedIdentityCredential(&options) - if err != nil { - return nil, fmt.Errorf("failed to acquire MSI token: %w", err) - } - case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "": - // Workload Identity based authentication - var options azidentity.ManagedIdentityCredentialOptions - options.ID = azidentity.ClientID(opt.MSIClientID) - - msiCred, err := azidentity.NewManagedIdentityCredential(&options) - if err != nil { - return nil, fmt.Errorf("failed to acquire MSI token: %w", err) - } - - getClientAssertions := func(context.Context) (string, error) { - token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ - Scopes: []string{"api://AzureADTokenExchange"}, - }) - if err != nil { - return "", fmt.Errorf("failed to acquire MSI token: %w", err) - } - - return token.Token, nil - } - - assertOpts := &azidentity.ClientAssertionCredentialOptions{} - cred, err = azidentity.NewClientAssertionCredential( - opt.Tenant, - opt.ClientID, - getClientAssertions, - assertOpts) - - if err != nil { - return nil, fmt.Errorf("failed to acquire client assertion token: %w", err) - } - default: - return nil, errors.New("no authentication method configured") - } - - // Make the client if not already created - if client == nil { - // Work out what the endpoint is if it is still unset - if opt.Endpoint == "" { - if opt.Account == "" { - return nil, fmt.Errorf("account must be set: can't make service URL") - } - u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL)) - if err != nil { - return nil, fmt.Errorf("failed to make azure storage URL from account: %w", err) - } - opt.Endpoint = u.String() - } - if sharedKeyCred != nil { - // Shared key cred - client, err = service.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt) - if err != nil { - return nil, fmt.Errorf("create client with shared key failed: %w", err) - } - } else if cred != nil { - // Azidentity cred - client, err = service.NewClient(opt.Endpoint, cred, &clientOpt) - if err != nil { - return nil, fmt.Errorf("create client failed: %w", err) - } - } - } - if client == nil { - return nil, fmt.Errorf("internal error: auth failed to make credentials or client") - } - - shareClient := client.NewShareClient(opt.ShareName) + shareClient := res.Client.NewShareClient(opt.ShareName) svc := shareClient.NewRootDirectoryClient() f := &Fs{ shareClient: shareClient, diff --git a/backend/azurefiles/azurefiles_internal_test.go b/backend/azurefiles/azurefiles_internal_test.go index 4996267a1..bf96e9646 100644 --- a/backend/azurefiles/azurefiles_internal_test.go +++ b/backend/azurefiles/azurefiles_internal_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/rclone/rclone/backend/azureblob/auth" "github.com/rclone/rclone/fstest/fstests" "github.com/stretchr/testify/assert" ) @@ -28,22 +29,28 @@ func (f *Fs) InternalTestAuth(t *testing.T) { { name: "ConnectionString", options: &Options{ - ShareName: shareName, - ConnectionString: "", + ShareName: shareName, + Options: auth.Options{ + ConnectionString: "", + }, }, }, { name: "AccountAndKey", options: &Options{ ShareName: shareName, - Account: "", - Key: "", + Options: auth.Options{ + Account: "", + Key: "", + }, }}, { name: "SASUrl", options: &Options{ ShareName: shareName, - SASURL: "", + Options: auth.Options{ + SASURL: "", + }, }}, }