opendal/types/operator/
builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34///     // Create fs backend builder.
35///     let builder = Fs::default().root("/tmp");
36///
37///     // Build an `Operator` to start operating the storage.
38///     let op: Operator = Operator::new(builder)?.finish();
39///
40///     Ok(())
41/// }
42/// ```
43impl Operator {
44    /// Create a new operator with input builder.
45    ///
46    /// OpenDAL will call `builder.build()` internally, so we don't need
47    /// to import `opendal::Builder` trait.
48    ///
49    /// # Examples
50    ///
51    /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52    ///
53    /// ```
54    /// # use anyhow::Result;
55    /// use opendal::services::Fs;
56    /// use opendal::Operator;
57    /// async fn test() -> Result<()> {
58    ///     // Create fs backend builder.
59    ///     let builder = Fs::default().root("/tmp");
60    ///
61    ///     // Build an `Operator` to start operating the storage.
62    ///     let op: Operator = Operator::new(builder)?.finish();
63    ///
64    ///     Ok(())
65    /// }
66    /// ```
67    #[allow(clippy::new_ret_no_self)]
68    pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69        let acc = ab.build()?;
70        Ok(OperatorBuilder::new(acc))
71    }
72
73    /// Create a new operator from given config.
74    ///
75    /// # Examples
76    ///
77    /// ```
78    /// # use anyhow::Result;
79    /// use std::collections::HashMap;
80    ///
81    /// use opendal::services::MemoryConfig;
82    /// use opendal::Operator;
83    /// async fn test() -> Result<()> {
84    ///     let cfg = MemoryConfig::default();
85    ///
86    ///     // Build an `Operator` to start operating the storage.
87    ///     let op: Operator = Operator::from_config(cfg)?.finish();
88    ///
89    ///     Ok(())
90    /// }
91    /// ```
92    pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93        let builder = cfg.into_builder();
94        let acc = builder.build()?;
95        Ok(OperatorBuilder::new(acc))
96    }
97
98    /// Create a new operator from given iterator in static dispatch.
99    ///
100    /// # Notes
101    ///
102    /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103    ///
104    /// # Examples
105    ///
106    /// ```
107    /// # use anyhow::Result;
108    /// use std::collections::HashMap;
109    ///
110    /// use opendal::services::Fs;
111    /// use opendal::Operator;
112    /// async fn test() -> Result<()> {
113    ///     let map = HashMap::from([
114    ///         // Set the root for fs, all operations will happen under this root.
115    ///         //
116    ///         // NOTE: the root must be absolute path.
117    ///         ("root".to_string(), "/tmp".to_string()),
118    ///     ]);
119    ///
120    ///     // Build an `Operator` to start operating the storage.
121    ///     let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122    ///
123    ///     Ok(())
124    /// }
125    /// ```
126    #[allow(clippy::should_implement_trait)]
127    pub fn from_iter<B: Builder>(
128        iter: impl IntoIterator<Item = (String, String)>,
129    ) -> Result<OperatorBuilder<impl Access>> {
130        let builder = B::Config::from_iter(iter)?.into_builder();
131        let acc = builder.build()?;
132        Ok(OperatorBuilder::new(acc))
133    }
134
135    /// Create a new operator by parsing configuration from a URI.
136    ///
137    /// # Examples
138    ///
139    /// ```
140    /// # use anyhow::Result;
141    /// use opendal::Operator;
142    ///
143    /// # fn example() -> Result<()> {
144    /// let op = Operator::from_uri("memory://localhost/", [])?;
145    /// # let _ = op;
146    /// # Ok(())
147    /// # }
148    /// ```
149    pub fn from_uri(
150        uri: &str,
151        options: impl IntoIterator<Item = (String, String)>,
152    ) -> Result<Operator> {
153        crate::DEFAULT_OPERATOR_REGISTRY.load(uri, options)
154    }
155
156    /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
157    ///
158    /// # Notes
159    ///
160    /// `via_iter` generates a `Operator` which allows building operator without generic type.
161    ///
162    /// # Examples
163    ///
164    /// ```
165    /// # use anyhow::Result;
166    /// use std::collections::HashMap;
167    ///
168    /// use opendal::Operator;
169    /// use opendal::Scheme;
170    /// async fn test() -> Result<()> {
171    ///     let map = [
172    ///         // Set the root for fs, all operations will happen under this root.
173    ///         //
174    ///         // NOTE: the root must be absolute path.
175    ///         ("root".to_string(), "/tmp".to_string()),
176    ///     ];
177    ///
178    ///     // Build an `Operator` to start operating the storage.
179    ///     let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
180    ///
181    ///     Ok(())
182    /// }
183    /// ```
184    #[allow(unused_variables, unreachable_code)]
185    pub fn via_iter(
186        scheme: Scheme,
187        iter: impl IntoIterator<Item = (String, String)>,
188    ) -> Result<Operator> {
189        let op = match scheme {
190            #[cfg(feature = "services-aliyun-drive")]
191            Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
192            #[cfg(feature = "services-alluxio")]
193            Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
194            #[cfg(feature = "services-cloudflare-kv")]
195            Scheme::CloudflareKv => Self::from_iter::<services::CloudflareKv>(iter)?.finish(),
196            #[cfg(feature = "services-compfs")]
197            Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
198            #[cfg(feature = "services-upyun")]
199            Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
200            #[cfg(feature = "services-koofr")]
201            Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
202            #[cfg(feature = "services-yandex-disk")]
203            Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
204            #[cfg(feature = "services-pcloud")]
205            Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
206            #[cfg(feature = "services-azblob")]
207            Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
208            #[cfg(feature = "services-azdls")]
209            Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
210            #[cfg(feature = "services-azfile")]
211            Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
212            #[cfg(feature = "services-b2")]
213            Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
214            #[cfg(feature = "services-cacache")]
215            Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
216            #[cfg(feature = "services-cos")]
217            Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
218            #[cfg(feature = "services-d1")]
219            Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
220            #[cfg(feature = "services-dashmap")]
221            Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
222            #[cfg(feature = "services-dropbox")]
223            Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
224            #[cfg(feature = "services-etcd")]
225            Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
226            #[cfg(feature = "services-foundationdb")]
227            Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
228            #[cfg(feature = "services-fs")]
229            Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
230            #[cfg(feature = "services-ftp")]
231            Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
232            #[cfg(feature = "services-gcs")]
233            Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
234            #[cfg(feature = "services-ghac")]
235            Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
236            #[cfg(feature = "services-gridfs")]
237            Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
238            #[cfg(feature = "services-github")]
239            Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
240            #[cfg(feature = "services-hdfs")]
241            Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
242            #[cfg(feature = "services-http")]
243            Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
244            #[cfg(feature = "services-huggingface")]
245            Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
246            #[cfg(feature = "services-ipfs")]
247            Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
248            #[cfg(feature = "services-ipmfs")]
249            Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
250            #[cfg(feature = "services-memcached")]
251            Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
252            #[cfg(feature = "services-memory")]
253            Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
254            #[cfg(feature = "services-mini-moka")]
255            Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
256            #[cfg(feature = "services-moka")]
257            Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
258            #[cfg(feature = "services-monoiofs")]
259            Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
260            #[cfg(feature = "services-mysql")]
261            Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
262            #[cfg(feature = "services-obs")]
263            Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
264            #[cfg(feature = "services-onedrive")]
265            Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
266            #[cfg(feature = "services-postgresql")]
267            Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
268            #[cfg(feature = "services-gdrive")]
269            Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
270            #[cfg(feature = "services-oss")]
271            Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
272            #[cfg(feature = "services-persy")]
273            Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
274            #[cfg(feature = "services-redis")]
275            Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
276            #[cfg(feature = "services-rocksdb")]
277            Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
278            #[cfg(feature = "services-s3")]
279            Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
280            #[cfg(feature = "services-seafile")]
281            Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
282            #[cfg(feature = "services-sftp")]
283            Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
284            #[cfg(feature = "services-sled")]
285            Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
286            #[cfg(feature = "services-sqlite")]
287            Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
288            #[cfg(feature = "services-swift")]
289            Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
290            #[cfg(feature = "services-tikv")]
291            Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
292            #[cfg(feature = "services-vercel-artifacts")]
293            Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
294            #[cfg(feature = "services-vercel-blob")]
295            Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
296            #[cfg(feature = "services-webdav")]
297            Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
298            #[cfg(feature = "services-webhdfs")]
299            Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
300            #[cfg(feature = "services-redb")]
301            Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
302            #[cfg(feature = "services-mongodb")]
303            Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
304            #[cfg(feature = "services-hdfs-native")]
305            Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
306            #[cfg(feature = "services-lakefs")]
307            Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
308            v => {
309                return Err(Error::new(
310                    ErrorKind::Unsupported,
311                    "scheme is not enabled or supported",
312                )
313                .with_context("scheme", v))
314            }
315        };
316
317        Ok(op)
318    }
319
320    /// Create a new operator from given map.
321    ///
322    /// # Notes
323    ///
324    /// from_map is using static dispatch layers which is zero cost. via_map is
325    /// using dynamic dispatch layers which has a bit runtime overhead with an
326    /// extra vtable lookup and unable to inline. But from_map requires generic
327    /// type parameter which is not always easy to be used.
328    ///
329    /// # Examples
330    ///
331    /// ```
332    /// # use anyhow::Result;
333    /// use std::collections::HashMap;
334    ///
335    /// use opendal::services::Fs;
336    /// use opendal::Operator;
337    /// async fn test() -> Result<()> {
338    ///     let map = HashMap::from([
339    ///         // Set the root for fs, all operations will happen under this root.
340    ///         //
341    ///         // NOTE: the root must be absolute path.
342    ///         ("root".to_string(), "/tmp".to_string()),
343    ///     ]);
344    ///
345    ///     // Build an `Operator` to start operating the storage.
346    ///     let op: Operator = Operator::from_map::<Fs>(map)?.finish();
347    ///
348    ///     Ok(())
349    /// }
350    /// ```
351    #[deprecated = "use from_iter instead"]
352    pub fn from_map<B: Builder>(
353        map: HashMap<String, String>,
354    ) -> Result<OperatorBuilder<impl Access>> {
355        Self::from_iter::<B>(map)
356    }
357
358    /// Create a new operator from given scheme and map.
359    ///
360    /// # Notes
361    ///
362    /// from_map is using static dispatch layers which is zero cost. via_map is
363    /// using dynamic dispatch layers which has a bit runtime overhead with an
364    /// extra vtable lookup and unable to inline. But from_map requires generic
365    /// type parameter which is not always easy to be used.
366    ///
367    /// # Examples
368    ///
369    /// ```
370    /// # use anyhow::Result;
371    /// use std::collections::HashMap;
372    ///
373    /// use opendal::Operator;
374    /// use opendal::Scheme;
375    /// async fn test() -> Result<()> {
376    ///     let map = HashMap::from([
377    ///         // Set the root for fs, all operations will happen under this root.
378    ///         //
379    ///         // NOTE: the root must be absolute path.
380    ///         ("root".to_string(), "/tmp".to_string()),
381    ///     ]);
382    ///
383    ///     // Build an `Operator` to start operating the storage.
384    ///     let op: Operator = Operator::via_map(Scheme::Fs, map)?;
385    ///
386    ///     Ok(())
387    /// }
388    /// ```
389    #[deprecated = "use via_iter instead"]
390    pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
391        Self::via_iter(scheme, map)
392    }
393
394    /// Create a new layer with dynamic dispatch.
395    ///
396    /// Please note that `Layer` can modify internal contexts such as `HttpClient`
397    /// and `Runtime` for the operator. Therefore, it is recommended to add layers
398    /// before interacting with the storage. Adding or duplicating layers after
399    /// accessing the storage may result in unexpected behavior.
400    ///
401    /// # Notes
402    ///
403    /// `OperatorBuilder::layer()` is using static dispatch which is zero
404    /// cost. `Operator::layer()` is using dynamic dispatch which has a
405    /// bit runtime overhead with an extra vtable lookup and unable to
406    /// inline.
407    ///
408    /// It's always recommended to use `OperatorBuilder::layer()` instead.
409    ///
410    /// # Examples
411    ///
412    /// ```no_run
413    /// # use std::sync::Arc;
414    /// # use anyhow::Result;
415    /// use opendal::layers::LoggingLayer;
416    /// use opendal::services::Fs;
417    /// use opendal::Operator;
418    ///
419    /// # async fn test() -> Result<()> {
420    /// let op = Operator::new(Fs::default())?.finish();
421    /// let op = op.layer(LoggingLayer::default());
422    /// // All operations will go through the new_layer
423    /// let _ = op.read("test_file").await?;
424    /// # Ok(())
425    /// # }
426    /// ```
427    #[must_use]
428    pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
429        Self::from_inner(Arc::new(
430            TypeEraseLayer.layer(layer.layer(self.into_inner())),
431        ))
432    }
433}
434
435/// OperatorBuilder is a typed builder to build an Operator.
436///
437/// # Notes
438///
439/// OpenDAL uses static dispatch internally and only performs dynamic
440/// dispatch at the outmost type erase layer. OperatorBuilder is the only
441/// public API provided by OpenDAL come with generic parameters.
442///
443/// It's required to call `finish` after the operator built.
444///
445/// # Examples
446///
447/// For users who want to support many services, we can build a helper function like the following:
448///
449/// ```
450/// use std::collections::HashMap;
451///
452/// use opendal::layers::LoggingLayer;
453/// use opendal::layers::RetryLayer;
454/// use opendal::services;
455/// use opendal::Builder;
456/// use opendal::Operator;
457/// use opendal::Result;
458/// use opendal::Scheme;
459///
460/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
461///     let op = Operator::from_map::<B>(cfg)?
462///         .layer(LoggingLayer::default())
463///         .layer(RetryLayer::new())
464///         .finish();
465///
466///     Ok(op)
467/// }
468///
469/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
470///     let _ = match scheme {
471///         Scheme::S3 => init_service::<services::S3>(cfg)?,
472///         Scheme::Fs => init_service::<services::Fs>(cfg)?,
473///         _ => todo!(),
474///     };
475///
476///     Ok(())
477/// }
478/// ```
479pub struct OperatorBuilder<A: Access> {
480    accessor: A,
481}
482
483impl<A: Access> OperatorBuilder<A> {
484    /// Create a new operator builder.
485    #[allow(clippy::new_ret_no_self)]
486    pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
487        // Make sure error context layer has been attached.
488        OperatorBuilder { accessor }
489            .layer(ErrorContextLayer)
490            .layer(CompleteLayer)
491            .layer(CorrectnessCheckLayer)
492    }
493
494    /// Create a new layer with static dispatch.
495    ///
496    /// # Notes
497    ///
498    /// `OperatorBuilder::layer()` is using static dispatch which is zero
499    /// cost. `Operator::layer()` is using dynamic dispatch which has a
500    /// bit runtime overhead with an extra vtable lookup and unable to
501    /// inline.
502    ///
503    /// It's always recommended to use `OperatorBuilder::layer()` instead.
504    ///
505    /// # Examples
506    ///
507    /// ```no_run
508    /// # use std::sync::Arc;
509    /// # use anyhow::Result;
510    /// use opendal::layers::LoggingLayer;
511    /// use opendal::services::Fs;
512    /// use opendal::Operator;
513    ///
514    /// # async fn test() -> Result<()> {
515    /// let op = Operator::new(Fs::default())?
516    ///     .layer(LoggingLayer::default())
517    ///     .finish();
518    /// // All operations will go through the new_layer
519    /// let _ = op.read("test_file").await?;
520    /// # Ok(())
521    /// # }
522    /// ```
523    #[must_use]
524    pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
525        OperatorBuilder {
526            accessor: layer.layer(self.accessor),
527        }
528    }
529
530    /// Finish the building to construct an Operator.
531    pub fn finish(self) -> Operator {
532        let ob = self.layer(TypeEraseLayer);
533        Operator::from_inner(Arc::new(ob.accessor) as Accessor)
534    }
535}
opendal/types/operator/builder.rs

opendal/types/operator/
builder.rs
