opendal/types/operator/builder.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34/// // Create fs backend builder.
35/// let builder = Fs::default().root("/tmp");
36///
37/// // Build an `Operator` to start operating the storage.
38/// let op: Operator = Operator::new(builder)?.finish();
39///
40/// Ok(())
41/// }
42/// ```
43impl Operator {
44 /// Create a new operator with input builder.
45 ///
46 /// OpenDAL will call `builder.build()` internally, so we don't need
47 /// to import `opendal::Builder` trait.
48 ///
49 /// # Examples
50 ///
51 /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52 ///
53 /// ```
54 /// # use anyhow::Result;
55 /// use opendal::services::Fs;
56 /// use opendal::Operator;
57 /// async fn test() -> Result<()> {
58 /// // Create fs backend builder.
59 /// let builder = Fs::default().root("/tmp");
60 ///
61 /// // Build an `Operator` to start operating the storage.
62 /// let op: Operator = Operator::new(builder)?.finish();
63 ///
64 /// Ok(())
65 /// }
66 /// ```
67 #[allow(clippy::new_ret_no_self)]
68 pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69 let acc = ab.build()?;
70 Ok(OperatorBuilder::new(acc))
71 }
72
73 /// Create a new operator from given config.
74 ///
75 /// # Examples
76 ///
77 /// ```
78 /// # use anyhow::Result;
79 /// use std::collections::HashMap;
80 ///
81 /// use opendal::services::MemoryConfig;
82 /// use opendal::Operator;
83 /// async fn test() -> Result<()> {
84 /// let cfg = MemoryConfig::default();
85 ///
86 /// // Build an `Operator` to start operating the storage.
87 /// let op: Operator = Operator::from_config(cfg)?.finish();
88 ///
89 /// Ok(())
90 /// }
91 /// ```
92 pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93 let builder = cfg.into_builder();
94 let acc = builder.build()?;
95 Ok(OperatorBuilder::new(acc))
96 }
97
98 /// Create a new operator from given iterator in static dispatch.
99 ///
100 /// # Notes
101 ///
102 /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103 ///
104 /// # Examples
105 ///
106 /// ```
107 /// # use anyhow::Result;
108 /// use std::collections::HashMap;
109 ///
110 /// use opendal::services::Fs;
111 /// use opendal::Operator;
112 /// async fn test() -> Result<()> {
113 /// let map = HashMap::from([
114 /// // Set the root for fs, all operations will happen under this root.
115 /// //
116 /// // NOTE: the root must be absolute path.
117 /// ("root".to_string(), "/tmp".to_string()),
118 /// ]);
119 ///
120 /// // Build an `Operator` to start operating the storage.
121 /// let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122 ///
123 /// Ok(())
124 /// }
125 /// ```
126 #[allow(clippy::should_implement_trait)]
127 pub fn from_iter<B: Builder>(
128 iter: impl IntoIterator<Item = (String, String)>,
129 ) -> Result<OperatorBuilder<impl Access>> {
130 let builder = B::Config::from_iter(iter)?.into_builder();
131 let acc = builder.build()?;
132 Ok(OperatorBuilder::new(acc))
133 }
134
135 /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
136 ///
137 /// # Notes
138 ///
139 /// `via_iter` generates a `Operator` which allows building operator without generic type.
140 ///
141 /// # Examples
142 ///
143 /// ```
144 /// # use anyhow::Result;
145 /// use std::collections::HashMap;
146 ///
147 /// use opendal::Operator;
148 /// use opendal::Scheme;
149 /// async fn test() -> Result<()> {
150 /// let map = [
151 /// // Set the root for fs, all operations will happen under this root.
152 /// //
153 /// // NOTE: the root must be absolute path.
154 /// ("root".to_string(), "/tmp".to_string()),
155 /// ];
156 ///
157 /// // Build an `Operator` to start operating the storage.
158 /// let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
159 ///
160 /// Ok(())
161 /// }
162 /// ```
163 #[allow(unused_variables, unreachable_code)]
164 pub fn via_iter(
165 scheme: Scheme,
166 iter: impl IntoIterator<Item = (String, String)>,
167 ) -> Result<Operator> {
168 let op = match scheme {
169 #[cfg(feature = "services-aliyun-drive")]
170 Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
171 #[cfg(feature = "services-alluxio")]
172 Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
173 #[cfg(feature = "services-cloudflare-kv")]
174 Scheme::CloudflareKv => Self::from_iter::<services::CloudflareKv>(iter)?.finish(),
175 #[cfg(feature = "services-compfs")]
176 Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
177 #[cfg(feature = "services-upyun")]
178 Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
179 #[cfg(feature = "services-koofr")]
180 Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
181 #[cfg(feature = "services-yandex-disk")]
182 Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
183 #[cfg(feature = "services-pcloud")]
184 Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
185 #[cfg(feature = "services-azblob")]
186 Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
187 #[cfg(feature = "services-azdls")]
188 Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
189 #[cfg(feature = "services-azfile")]
190 Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
191 #[cfg(feature = "services-b2")]
192 Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
193 #[cfg(feature = "services-cacache")]
194 Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
195 #[cfg(feature = "services-cos")]
196 Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
197 #[cfg(feature = "services-d1")]
198 Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
199 #[cfg(feature = "services-dashmap")]
200 Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
201 #[cfg(feature = "services-dropbox")]
202 Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
203 #[cfg(feature = "services-etcd")]
204 Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
205 #[cfg(feature = "services-foundationdb")]
206 Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
207 #[cfg(feature = "services-fs")]
208 Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
209 #[cfg(feature = "services-ftp")]
210 Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
211 #[cfg(feature = "services-gcs")]
212 Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
213 #[cfg(feature = "services-ghac")]
214 Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
215 #[cfg(feature = "services-gridfs")]
216 Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
217 #[cfg(feature = "services-github")]
218 Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
219 #[cfg(feature = "services-hdfs")]
220 Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
221 #[cfg(feature = "services-http")]
222 Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
223 #[cfg(feature = "services-huggingface")]
224 Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
225 #[cfg(feature = "services-ipfs")]
226 Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
227 #[cfg(feature = "services-ipmfs")]
228 Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
229 #[cfg(feature = "services-memcached")]
230 Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
231 #[cfg(feature = "services-memory")]
232 Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
233 #[cfg(feature = "services-mini-moka")]
234 Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
235 #[cfg(feature = "services-moka")]
236 Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
237 #[cfg(feature = "services-monoiofs")]
238 Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
239 #[cfg(feature = "services-mysql")]
240 Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
241 #[cfg(feature = "services-obs")]
242 Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
243 #[cfg(feature = "services-onedrive")]
244 Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
245 #[cfg(feature = "services-postgresql")]
246 Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
247 #[cfg(feature = "services-gdrive")]
248 Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
249 #[cfg(feature = "services-oss")]
250 Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
251 #[cfg(feature = "services-persy")]
252 Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
253 #[cfg(feature = "services-redis")]
254 Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
255 #[cfg(feature = "services-rocksdb")]
256 Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
257 #[cfg(feature = "services-s3")]
258 Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
259 #[cfg(feature = "services-seafile")]
260 Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
261 #[cfg(feature = "services-sftp")]
262 Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
263 #[cfg(feature = "services-sled")]
264 Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
265 #[cfg(feature = "services-sqlite")]
266 Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
267 #[cfg(feature = "services-swift")]
268 Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
269 #[cfg(feature = "services-tikv")]
270 Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
271 #[cfg(feature = "services-vercel-artifacts")]
272 Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
273 #[cfg(feature = "services-vercel-blob")]
274 Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
275 #[cfg(feature = "services-webdav")]
276 Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
277 #[cfg(feature = "services-webhdfs")]
278 Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
279 #[cfg(feature = "services-redb")]
280 Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
281 #[cfg(feature = "services-mongodb")]
282 Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
283 #[cfg(feature = "services-hdfs-native")]
284 Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
285 #[cfg(feature = "services-lakefs")]
286 Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
287 v => {
288 return Err(Error::new(
289 ErrorKind::Unsupported,
290 "scheme is not enabled or supported",
291 )
292 .with_context("scheme", v))
293 }
294 };
295
296 Ok(op)
297 }
298
299 /// Create a new operator from given map.
300 ///
301 /// # Notes
302 ///
303 /// from_map is using static dispatch layers which is zero cost. via_map is
304 /// using dynamic dispatch layers which has a bit runtime overhead with an
305 /// extra vtable lookup and unable to inline. But from_map requires generic
306 /// type parameter which is not always easy to be used.
307 ///
308 /// # Examples
309 ///
310 /// ```
311 /// # use anyhow::Result;
312 /// use std::collections::HashMap;
313 ///
314 /// use opendal::services::Fs;
315 /// use opendal::Operator;
316 /// async fn test() -> Result<()> {
317 /// let map = HashMap::from([
318 /// // Set the root for fs, all operations will happen under this root.
319 /// //
320 /// // NOTE: the root must be absolute path.
321 /// ("root".to_string(), "/tmp".to_string()),
322 /// ]);
323 ///
324 /// // Build an `Operator` to start operating the storage.
325 /// let op: Operator = Operator::from_map::<Fs>(map)?.finish();
326 ///
327 /// Ok(())
328 /// }
329 /// ```
330 #[deprecated = "use from_iter instead"]
331 pub fn from_map<B: Builder>(
332 map: HashMap<String, String>,
333 ) -> Result<OperatorBuilder<impl Access>> {
334 Self::from_iter::<B>(map)
335 }
336
337 /// Create a new operator from given scheme and map.
338 ///
339 /// # Notes
340 ///
341 /// from_map is using static dispatch layers which is zero cost. via_map is
342 /// using dynamic dispatch layers which has a bit runtime overhead with an
343 /// extra vtable lookup and unable to inline. But from_map requires generic
344 /// type parameter which is not always easy to be used.
345 ///
346 /// # Examples
347 ///
348 /// ```
349 /// # use anyhow::Result;
350 /// use std::collections::HashMap;
351 ///
352 /// use opendal::Operator;
353 /// use opendal::Scheme;
354 /// async fn test() -> Result<()> {
355 /// let map = HashMap::from([
356 /// // Set the root for fs, all operations will happen under this root.
357 /// //
358 /// // NOTE: the root must be absolute path.
359 /// ("root".to_string(), "/tmp".to_string()),
360 /// ]);
361 ///
362 /// // Build an `Operator` to start operating the storage.
363 /// let op: Operator = Operator::via_map(Scheme::Fs, map)?;
364 ///
365 /// Ok(())
366 /// }
367 /// ```
368 #[deprecated = "use via_iter instead"]
369 pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
370 Self::via_iter(scheme, map)
371 }
372
373 /// Create a new layer with dynamic dispatch.
374 ///
375 /// Please note that `Layer` can modify internal contexts such as `HttpClient`
376 /// and `Runtime` for the operator. Therefore, it is recommended to add layers
377 /// before interacting with the storage. Adding or duplicating layers after
378 /// accessing the storage may result in unexpected behavior.
379 ///
380 /// # Notes
381 ///
382 /// `OperatorBuilder::layer()` is using static dispatch which is zero
383 /// cost. `Operator::layer()` is using dynamic dispatch which has a
384 /// bit runtime overhead with an extra vtable lookup and unable to
385 /// inline.
386 ///
387 /// It's always recommended to use `OperatorBuilder::layer()` instead.
388 ///
389 /// # Examples
390 ///
391 /// ```no_run
392 /// # use std::sync::Arc;
393 /// # use anyhow::Result;
394 /// use opendal::layers::LoggingLayer;
395 /// use opendal::services::Fs;
396 /// use opendal::Operator;
397 ///
398 /// # async fn test() -> Result<()> {
399 /// let op = Operator::new(Fs::default())?.finish();
400 /// let op = op.layer(LoggingLayer::default());
401 /// // All operations will go through the new_layer
402 /// let _ = op.read("test_file").await?;
403 /// # Ok(())
404 /// # }
405 /// ```
406 #[must_use]
407 pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
408 Self::from_inner(Arc::new(
409 TypeEraseLayer.layer(layer.layer(self.into_inner())),
410 ))
411 }
412}
413
414/// OperatorBuilder is a typed builder to build an Operator.
415///
416/// # Notes
417///
418/// OpenDAL uses static dispatch internally and only performs dynamic
419/// dispatch at the outmost type erase layer. OperatorBuilder is the only
420/// public API provided by OpenDAL come with generic parameters.
421///
422/// It's required to call `finish` after the operator built.
423///
424/// # Examples
425///
426/// For users who want to support many services, we can build a helper function like the following:
427///
428/// ```
429/// use std::collections::HashMap;
430///
431/// use opendal::layers::LoggingLayer;
432/// use opendal::layers::RetryLayer;
433/// use opendal::services;
434/// use opendal::Builder;
435/// use opendal::Operator;
436/// use opendal::Result;
437/// use opendal::Scheme;
438///
439/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
440/// let op = Operator::from_map::<B>(cfg)?
441/// .layer(LoggingLayer::default())
442/// .layer(RetryLayer::new())
443/// .finish();
444///
445/// Ok(op)
446/// }
447///
448/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
449/// let _ = match scheme {
450/// Scheme::S3 => init_service::<services::S3>(cfg)?,
451/// Scheme::Fs => init_service::<services::Fs>(cfg)?,
452/// _ => todo!(),
453/// };
454///
455/// Ok(())
456/// }
457/// ```
458pub struct OperatorBuilder<A: Access> {
459 accessor: A,
460}
461
462impl<A: Access> OperatorBuilder<A> {
463 /// Create a new operator builder.
464 #[allow(clippy::new_ret_no_self)]
465 pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
466 // Make sure error context layer has been attached.
467 OperatorBuilder { accessor }
468 .layer(ErrorContextLayer)
469 .layer(CompleteLayer)
470 .layer(CorrectnessCheckLayer)
471 }
472
473 /// Create a new layer with static dispatch.
474 ///
475 /// # Notes
476 ///
477 /// `OperatorBuilder::layer()` is using static dispatch which is zero
478 /// cost. `Operator::layer()` is using dynamic dispatch which has a
479 /// bit runtime overhead with an extra vtable lookup and unable to
480 /// inline.
481 ///
482 /// It's always recommended to use `OperatorBuilder::layer()` instead.
483 ///
484 /// # Examples
485 ///
486 /// ```no_run
487 /// # use std::sync::Arc;
488 /// # use anyhow::Result;
489 /// use opendal::layers::LoggingLayer;
490 /// use opendal::services::Fs;
491 /// use opendal::Operator;
492 ///
493 /// # async fn test() -> Result<()> {
494 /// let op = Operator::new(Fs::default())?
495 /// .layer(LoggingLayer::default())
496 /// .finish();
497 /// // All operations will go through the new_layer
498 /// let _ = op.read("test_file").await?;
499 /// # Ok(())
500 /// # }
501 /// ```
502 #[must_use]
503 pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
504 OperatorBuilder {
505 accessor: layer.layer(self.accessor),
506 }
507 }
508
509 /// Finish the building to construct an Operator.
510 pub fn finish(self) -> Operator {
511 let ob = self.layer(TypeEraseLayer);
512 Operator::from_inner(Arc::new(ob.accessor) as Accessor)
513 }
514}