opendal/services/s3/
config.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt::Debug;
19use std::fmt::Formatter;
20
21use serde::Deserialize;
22use serde::Serialize;
23
24/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
25#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
26#[serde(default)]
27#[non_exhaustive]
28pub struct S3Config {
29    /// root of this backend.
30    ///
31    /// All operations will happen under this root.
32    ///
33    /// default to `/` if not set.
34    pub root: Option<String>,
35    /// bucket name of this backend.
36    ///
37    /// required.
38    #[serde(alias = "aws_bucket", alias = "aws_bucket_name", alias = "bucket_name")]
39    pub bucket: String,
40    /// is bucket versioning enabled for this bucket
41    pub enable_versioning: bool,
42    /// endpoint of this backend.
43    ///
44    /// Endpoint must be full uri, e.g.
45    ///
46    /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
47    /// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
48    /// - Aliyun OSS: `https://{region}.aliyuncs.com`
49    /// - Tencent COS: `https://cos.{region}.myqcloud.com`
50    /// - Minio: `http://127.0.0.1:9000`
51    ///
52    /// If user inputs endpoint without scheme like "s3.amazonaws.com", we
53    /// will prepend "https://" before it.
54    ///
55    /// - If endpoint is set, we will take user's input first.
56    /// - If not, we will try to load it from environment.
57    /// - If still not set, default to `https://s3.amazonaws.com`.
58    #[serde(
59        alias = "aws_endpoint",
60        alias = "aws_endpoint_url",
61        alias = "endpoint_url"
62    )]
63    pub endpoint: Option<String>,
64    /// Region represent the signing region of this endpoint. This is required
65    /// if you are using the default AWS S3 endpoint.
66    ///
67    /// If using a custom endpoint,
68    /// - If region is set, we will take user's input first.
69    /// - If not, we will try to load it from environment.
70    #[serde(alias = "aws_region")]
71    pub region: Option<String>,
72
73    /// access_key_id of this backend.
74    ///
75    /// - If access_key_id is set, we will take user's input first.
76    /// - If not, we will try to load it from environment.
77    #[serde(alias = "aws_access_key_id")]
78    pub access_key_id: Option<String>,
79    /// secret_access_key of this backend.
80    ///
81    /// - If secret_access_key is set, we will take user's input first.
82    /// - If not, we will try to load it from environment.
83    #[serde(alias = "aws_secret_access_key")]
84    pub secret_access_key: Option<String>,
85    /// session_token (aka, security token) of this backend.
86    ///
87    /// This token will expire after sometime, it's recommended to set session_token
88    /// by hand.
89    #[serde(alias = "aws_session_token", alias = "aws_token", alias = "token")]
90    pub session_token: Option<String>,
91    /// role_arn for this backend.
92    ///
93    /// If `role_arn` is set, we will use already known config as source
94    /// credential to assume role with `role_arn`.
95    pub role_arn: Option<String>,
96    /// external_id for this backend.
97    pub external_id: Option<String>,
98    /// role_session_name for this backend.
99    pub role_session_name: Option<String>,
100    /// Disable config load so that opendal will not load config from
101    /// environment.
102    ///
103    /// For examples:
104    ///
105    /// - envs like `AWS_ACCESS_KEY_ID`
106    /// - files like `~/.aws/config`
107    pub disable_config_load: bool,
108    /// Disable load credential from ec2 metadata.
109    ///
110    /// This option is used to disable the default behavior of opendal
111    /// to load credential from ec2 metadata, a.k.a, IMDSv2
112    pub disable_ec2_metadata: bool,
113    /// Allow anonymous will allow opendal to send request without signing
114    /// when credential is not loaded.
115    pub allow_anonymous: bool,
116    /// server_side_encryption for this backend.
117    ///
118    /// Available values: `AES256`, `aws:kms`.
119    #[serde(alias = "aws_server_side_encryption")]
120    pub server_side_encryption: Option<String>,
121    /// server_side_encryption_aws_kms_key_id for this backend
122    ///
123    /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
124    ///   is not set, S3 will use aws managed kms key to encrypt data.
125    /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
126    ///   is a valid kms key id, S3 will use the provided kms key to encrypt data.
127    /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be
128    ///   returned.
129    /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id`
130    ///   is a noop.
131    #[serde(alias = "aws_sse_kms_key_id")]
132    pub server_side_encryption_aws_kms_key_id: Option<String>,
133    /// server_side_encryption_customer_algorithm for this backend.
134    ///
135    /// Available values: `AES256`.
136    pub server_side_encryption_customer_algorithm: Option<String>,
137    /// server_side_encryption_customer_key for this backend.
138    ///
139    /// Value: BASE64-encoded key that matches algorithm specified in
140    /// `server_side_encryption_customer_algorithm`.
141    #[serde(alias = "aws_sse_customer_key_base64")]
142    pub server_side_encryption_customer_key: Option<String>,
143    /// Set server_side_encryption_customer_key_md5 for this backend.
144    ///
145    /// Value: MD5 digest of key specified in `server_side_encryption_customer_key`.
146    pub server_side_encryption_customer_key_md5: Option<String>,
147    /// default storage_class for this backend.
148    ///
149    /// Available values:
150    /// - `DEEP_ARCHIVE`
151    /// - `GLACIER`
152    /// - `GLACIER_IR`
153    /// - `INTELLIGENT_TIERING`
154    /// - `ONEZONE_IA`
155    /// - `EXPRESS_ONEZONE`
156    /// - `OUTPOSTS`
157    /// - `REDUCED_REDUNDANCY`
158    /// - `STANDARD`
159    /// - `STANDARD_IA`
160    ///
161    /// S3 compatible services don't support all of them
162    pub default_storage_class: Option<String>,
163    /// Enable virtual host style so that opendal will send API requests
164    /// in virtual host style instead of path style.
165    ///
166    /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
167    /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
168    #[serde(
169        alias = "aws_virtual_hosted_style_request",
170        alias = "virtual_hosted_style_request"
171    )]
172    pub enable_virtual_host_style: bool,
173    /// Set maximum batch operations of this backend.
174    ///
175    /// Some compatible services have a limit on the number of operations in a batch request.
176    /// For example, R2 could return `Internal Error` while batch delete 1000 files.
177    ///
178    /// Please tune this value based on services' document.
179    #[deprecated(
180        since = "0.52.0",
181        note = "Please use `delete_max_size` instead of `batch_max_operations`"
182    )]
183    pub batch_max_operations: Option<usize>,
184    /// Set the maximum delete size of this backend.
185    ///
186    /// Some compatible services have a limit on the number of operations in a batch request.
187    /// For example, R2 could return `Internal Error` while batch delete 1000 files.
188    ///
189    /// Please tune this value based on services' document.
190    pub delete_max_size: Option<usize>,
191    /// Disable stat with override so that opendal will not send stat request with override queries.
192    ///
193    /// For example, R2 doesn't support stat with `response_content_type` query.
194    pub disable_stat_with_override: bool,
195    /// Checksum Algorithm to use when sending checksums in HTTP headers.
196    /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
197    ///
198    /// Available options:
199    /// - "crc32c"
200    #[serde(alias = "aws_checksum_algorithm")]
201    pub checksum_algorithm: Option<String>,
202    /// Disable write with if match so that opendal will not send write request with if match headers.
203    ///
204    /// For example, Ceph RADOS S3 doesn't support write with if match.
205    pub disable_write_with_if_match: bool,
206
207    /// Enable write with append so that opendal will send write request with append headers.
208    pub enable_write_with_append: bool,
209
210    /// OpenDAL uses List Objects V2 by default to list objects.
211    /// However, some legacy services do not yet support V2.
212    /// This option allows users to switch back to the older List Objects V1.
213    pub disable_list_objects_v2: bool,
214
215    /// Indicates whether the client agrees to pay for the requests made to the S3 bucket.
216    #[serde(alias = "aws_request_payer", alias = "request_payer")]
217    pub enable_request_payer: bool,
218}
219
220impl Debug for S3Config {
221    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
222        let mut d = f.debug_struct("S3Config");
223
224        d.field("root", &self.root)
225            .field("bucket", &self.bucket)
226            .field("endpoint", &self.endpoint)
227            .field("region", &self.region);
228
229        d.finish_non_exhaustive()
230    }
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    #[test]
238    fn test_s3_config_original_field_names() {
239        let json = r#"{
240            "bucket": "test-bucket",
241            "access_key_id": "test-key",
242            "secret_access_key": "test-secret",
243            "region": "us-west-2",
244            "endpoint": "https://s3.amazonaws.com",
245            "session_token": "test-token"
246        }"#;
247
248        let config: S3Config = serde_json::from_str(json).unwrap();
249        assert_eq!(config.bucket, "test-bucket");
250        assert_eq!(config.access_key_id, Some("test-key".to_string()));
251        assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
252        assert_eq!(config.region, Some("us-west-2".to_string()));
253        assert_eq!(
254            config.endpoint,
255            Some("https://s3.amazonaws.com".to_string())
256        );
257        assert_eq!(config.session_token, Some("test-token".to_string()));
258    }
259
260    #[test]
261    fn test_s3_config_aws_prefixed_aliases() {
262        let json = r#"{
263            "aws_bucket": "test-bucket",
264            "aws_access_key_id": "test-key",
265            "aws_secret_access_key": "test-secret",
266            "aws_region": "us-west-2",
267            "aws_endpoint": "https://s3.amazonaws.com",
268            "aws_session_token": "test-token"
269        }"#;
270
271        let config: S3Config = serde_json::from_str(json).unwrap();
272        assert_eq!(config.bucket, "test-bucket");
273        assert_eq!(config.access_key_id, Some("test-key".to_string()));
274        assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
275        assert_eq!(config.region, Some("us-west-2".to_string()));
276        assert_eq!(
277            config.endpoint,
278            Some("https://s3.amazonaws.com".to_string())
279        );
280        assert_eq!(config.session_token, Some("test-token".to_string()));
281    }
282
283    #[test]
284    fn test_s3_config_additional_aliases() {
285        let json = r#"{
286            "bucket_name": "test-bucket",
287            "token": "test-token",
288            "endpoint_url": "https://s3.amazonaws.com",
289            "virtual_hosted_style_request": true,
290            "aws_checksum_algorithm": "crc32c",
291            "request_payer": true
292        }"#;
293
294        let config: S3Config = serde_json::from_str(json).unwrap();
295        assert_eq!(config.bucket, "test-bucket");
296        assert_eq!(config.session_token, Some("test-token".to_string()));
297        assert_eq!(
298            config.endpoint,
299            Some("https://s3.amazonaws.com".to_string())
300        );
301        assert!(config.enable_virtual_host_style);
302        assert_eq!(config.checksum_algorithm, Some("crc32c".to_string()));
303        assert!(config.enable_request_payer);
304    }
305
306    #[test]
307    fn test_s3_config_encryption_aliases() {
308        let json = r#"{
309            "bucket": "test-bucket",
310            "aws_server_side_encryption": "aws:kms",
311            "aws_sse_kms_key_id": "test-kms-key",
312            "aws_sse_customer_key_base64": "dGVzdC1jdXN0b21lci1rZXk="
313        }"#;
314
315        let config: S3Config = serde_json::from_str(json).unwrap();
316        assert_eq!(config.bucket, "test-bucket");
317        assert_eq!(config.server_side_encryption, Some("aws:kms".to_string()));
318        assert_eq!(
319            config.server_side_encryption_aws_kms_key_id,
320            Some("test-kms-key".to_string())
321        );
322        assert_eq!(
323            config.server_side_encryption_customer_key,
324            Some("dGVzdC1jdXN0b21lci1rZXk=".to_string())
325        );
326    }
327}
opendal/services/s3/config.rs

opendal/services/s3/
config.rs
