opendal_service_hf/
config.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use super::backend::HfBuilder;
19use super::core::HfDownloadMode;
20use super::core::HfRepoType;
21use super::core::HfUri;
22use serde::Deserialize;
23use serde::Serialize;
24use std::fmt::Debug;
25
26use super::HUGGINGFACE_SCHEME;
27
28/// Configuration for Hugging Face service support.
29#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
30#[serde(default)]
31#[non_exhaustive]
32pub struct HfConfig {
33    /// Repo type of this backend. Required.
34    pub repo_type: Option<HfRepoType>,
35    /// Repo id of this backend.
36    ///
37    /// This is required.
38    pub repo_id: Option<String>,
39    /// Revision of this backend.
40    ///
41    /// Default is main.
42    pub revision: Option<String>,
43    /// Root of this backend. Can be "/path/to/dir".
44    ///
45    /// Default is "/".
46    pub root: Option<String>,
47    /// Token of this backend.
48    ///
49    /// This is optional.
50    pub token: Option<String>,
51    /// Endpoint of the Hugging Face Hub.
52    ///
53    /// The default is `https://huggingface.co`.
54    pub endpoint: Option<String>,
55    /// Download mode. Either `xet` (default) or `http`.
56    ///
57    /// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`
58    /// environment variable: a non-empty value forces `http`, otherwise it
59    /// defaults to `xet`. An explicit value here takes precedence.
60    ///
61    /// See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
62    pub download_mode: Option<HfDownloadMode>,
63    /// Enable caching of resolved HTTP download addresses and XET file metadata.
64    ///
65    /// Defaults to `false`. Set to `true` to share resolve results across readers
66    /// on the same backend. Changed files may remain invisible while cached
67    /// results are reused. A reader retains XET metadata from its first read for
68    /// its lifetime. Create a new reader to resolve the path again when this
69    /// option is disabled.
70    /// See [`HfBuilder::enable_resolve_cache`] for freshness semantics.
71    pub enable_resolve_cache: bool,
72}
73
74impl Debug for HfConfig {
75    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76        f.debug_struct("HfConfig")
77            .field(
78                "repo_type",
79                &self.repo_type.as_ref().map(HfRepoType::as_str),
80            )
81            .field("repo_id", &self.repo_id)
82            .field("revision", &self.revision)
83            .field("root", &self.root)
84            .field("download_mode", &self.download_mode)
85            .field("enable_resolve_cache", &self.enable_resolve_cache)
86            .finish_non_exhaustive()
87    }
88}
89
90impl opendal_core::Configurator for HfConfig {
91    type Builder = HfBuilder;
92
93    fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> {
94        let opts = uri.options();
95
96        // Reconstruct the full path from authority (name) and root.
97        // OperatorUri splits "hf://datasets/user/repo@rev/path" into
98        // name="datasets" and root="user/repo@rev/path".
99        let mut path = String::new();
100        if let Some(name) = uri.name()
101            && !name.is_empty()
102        {
103            path.push_str(name);
104        }
105        if let Some(root) = uri.root()
106            && !root.is_empty()
107        {
108            if !path.is_empty() {
109                path.push('/');
110            }
111            path.push_str(root);
112        }
113
114        let download_mode = opts
115            .get("download_mode")
116            .map(|s| HfDownloadMode::parse(s))
117            .transpose()?;
118
119        let enable_resolve_cache = opts
120            .get("enable_resolve_cache")
121            .map(|value| value.parse::<bool>())
122            .transpose()
123            .map_err(|err| {
124                opendal_core::Error::new(
125                    opendal_core::ErrorKind::ConfigInvalid,
126                    "enable_resolve_cache must be true or false",
127                )
128                .set_source(err)
129            })?
130            .unwrap_or_default();
131
132        if !path.is_empty() {
133            // Full URI like "hf://datasets/user/repo@rev/path"
134            let parsed = HfUri::parse(&path)?;
135            Ok(Self {
136                repo_type: Some(parsed.repo.repo_type),
137                repo_id: Some(parsed.repo.repo_id),
138                revision: parsed.repo.revision,
139                root: opts.get("root").cloned(),
140                token: opts.get("token").cloned(),
141                endpoint: opts.get("endpoint").cloned(),
142                download_mode,
143                enable_resolve_cache,
144            })
145        } else {
146            // Bare scheme from via_iter, all config is in options.
147            let repo_type = opts
148                .get("repo_type")
149                .ok_or_else(|| {
150                    opendal_core::Error::new(
151                        opendal_core::ErrorKind::ConfigInvalid,
152                        "repo_type is required",
153                    )
154                    .with_context("service", HUGGINGFACE_SCHEME)
155                })
156                .and_then(|s| HfRepoType::parse(s))?;
157            Ok(Self {
158                repo_type: Some(repo_type),
159                repo_id: opts.get("repo_id").cloned(),
160                revision: opts.get("revision").cloned(),
161                root: opts.get("root").cloned(),
162                token: opts.get("token").cloned(),
163                endpoint: opts.get("endpoint").cloned(),
164                download_mode,
165                enable_resolve_cache,
166            })
167        }
168    }
169
170    fn into_builder(self) -> Self::Builder {
171        HfBuilder { config: self }
172    }
173}
174
175#[cfg(test)]
176mod tests {
177    use super::*;
178    use opendal_core::Configurator;
179    use opendal_core::OperatorUri;
180
181    #[test]
182    fn from_uri_with_all_components() {
183        let uri = OperatorUri::new(
184            "hf://datasets/username/my_dataset@dev/train/data.csv",
185            Vec::<(String, String)>::new(),
186        )
187        .unwrap();
188
189        let cfg = HfConfig::from_uri(&uri).unwrap();
190        assert_eq!(cfg.repo_type, Some(HfRepoType::Dataset));
191        assert_eq!(cfg.repo_id.as_deref(), Some("username/my_dataset"));
192        assert_eq!(cfg.revision.as_deref(), Some("dev"));
193        assert!(cfg.root.is_none());
194    }
195
196    #[test]
197    fn from_uri_via_iter_options() {
198        // Simulates the via_iter path: bare scheme with options map.
199        let uri = OperatorUri::new(
200            "huggingface",
201            vec![
202                ("repo_type".to_string(), "dataset".to_string()),
203                (
204                    "repo_id".to_string(),
205                    "opendal/huggingface-testdata".to_string(),
206                ),
207                ("revision".to_string(), "main".to_string()),
208                ("root".to_string(), "/testdata/".to_string()),
209            ],
210        )
211        .unwrap();
212
213        let cfg = HfConfig::from_uri(&uri).unwrap();
214        assert_eq!(cfg.repo_type, Some(HfRepoType::Dataset));
215        assert_eq!(cfg.repo_id.as_deref(), Some("opendal/huggingface-testdata"));
216        assert_eq!(cfg.revision.as_deref(), Some("main"));
217        assert_eq!(cfg.root.as_deref(), Some("/testdata/"));
218    }
219
220    #[test]
221    fn from_uri_download_mode_http() {
222        let uri = OperatorUri::new(
223            "huggingface",
224            vec![
225                ("repo_type".to_string(), "dataset".to_string()),
226                ("repo_id".to_string(), "user/repo".to_string()),
227                ("download_mode".to_string(), "http".to_string()),
228            ],
229        )
230        .unwrap();
231
232        let cfg = HfConfig::from_uri(&uri).unwrap();
233        assert_eq!(cfg.download_mode, Some(HfDownloadMode::Http));
234    }
235
236    #[test]
237    fn from_uri_download_mode_defaults_to_xet() {
238        let uri = OperatorUri::new(
239            "huggingface",
240            vec![
241                ("repo_type".to_string(), "model".to_string()),
242                ("repo_id".to_string(), "user/repo".to_string()),
243            ],
244        )
245        .unwrap();
246
247        let cfg = HfConfig::from_uri(&uri).unwrap();
248        assert_eq!(cfg.download_mode.unwrap_or_default(), HfDownloadMode::Xet);
249    }
250
251    #[test]
252    fn from_uri_resolve_cache_options() {
253        for scheme in ["hf://datasets/user/repo", "huggingface"] {
254            for (value, expected) in [(None, false), (Some("false"), false), (Some("true"), true)] {
255                let mut options = vec![("repo_type", "dataset"), ("repo_id", "user/repo")];
256                if let Some(value) = value {
257                    options.push(("enable_resolve_cache", value));
258                }
259                let uri = OperatorUri::new(
260                    scheme,
261                    options
262                        .into_iter()
263                        .map(|(key, value)| (key.to_string(), value.to_string())),
264                )
265                .unwrap();
266                assert_eq!(
267                    HfConfig::from_uri(&uri).unwrap().enable_resolve_cache,
268                    expected
269                );
270            }
271
272            let uri = OperatorUri::new(
273                scheme,
274                [("enable_resolve_cache".to_string(), "invalid".to_string())],
275            )
276            .unwrap();
277            let err = HfConfig::from_uri(&uri).unwrap_err();
278            assert_eq!(err.kind(), opendal_core::ErrorKind::ConfigInvalid);
279            assert!(
280                err.to_string()
281                    .contains("enable_resolve_cache must be true or false")
282            );
283        }
284    }
285}
opendal_service_hf/config.rs

opendal_service_hf/
config.rs
