wit_deps/
lib.rs

1//! WIT dependency management core library
2
3#![forbid(clippy::unwrap_used)]
4#![warn(missing_docs)]
5
6mod cache;
7mod digest;
8mod lock;
9mod manifest;
10
11pub use cache::{Cache, Local as LocalCache, Write as WriteCache};
12pub use digest::{Digest, Reader as DigestReader, Writer as DigestWriter};
13pub use lock::{Entry as LockEntry, EntrySource as LockEntrySource, Lock};
14pub use manifest::{Entry as ManifestEntry, Manifest};
15
16pub use futures;
17pub use tokio;
18
19use core::array;
20
21use std::collections::{BTreeSet, HashMap, HashSet};
22use std::ffi::{OsStr, OsString};
23use std::path::{Path, PathBuf};
24
25use anyhow::Context;
26use futures::{try_join, FutureExt, Stream, TryStreamExt};
27use tokio::fs;
28use tokio::io::{AsyncRead, AsyncWrite};
29use tokio_stream::wrappers::ReadDirStream;
30use tracing::{debug, instrument, trace};
31
32/// WIT dependency identifier
33pub type Identifier = String;
34// TODO: Introduce a rich type with name validation
35//#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq)]
36//pub struct Identifier(String);
37
38fn is_wit(path: impl AsRef<Path>) -> bool {
39    path.as_ref()
40        .extension()
41        .is_some_and(|ext| ext.eq_ignore_ascii_case("wit"))
42}
43
44#[instrument(level = "trace", skip(path))]
45async fn remove_dir_all(path: impl AsRef<Path>) -> std::io::Result<()> {
46    let path = path.as_ref();
47    match fs::remove_dir_all(path).await {
48        Ok(()) => {
49            trace!("removed `{}`", path.display());
50            Ok(())
51        }
52        Err(e) => Err(std::io::Error::new(
53            e.kind(),
54            format!("failed to remove `{}`: {e}", path.display()),
55        )),
56    }
57}
58
59#[instrument(level = "trace", skip(path))]
60async fn recreate_dir(path: impl AsRef<Path>) -> std::io::Result<()> {
61    let path = path.as_ref();
62    match remove_dir_all(path).await {
63        Ok(()) => {}
64        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
65        Err(e) => return Err(e),
66    }
67    fs::create_dir_all(path)
68        .await
69        .map(|()| trace!("recreated `{}`", path.display()))
70        .map_err(|e| {
71            std::io::Error::new(
72                e.kind(),
73                format!("failed to create `{}`: {e}", path.display()),
74            )
75        })
76}
77
78/// Returns a stream of WIT file names within a directory at `path`
79#[instrument(level = "trace", skip(path))]
80async fn read_wits(
81    path: impl AsRef<Path>,
82) -> std::io::Result<impl Stream<Item = std::io::Result<OsString>>> {
83    let path = path.as_ref();
84    let st = fs::read_dir(path)
85        .await
86        .map(ReadDirStream::new)
87        .map_err(|e| {
88            std::io::Error::new(
89                e.kind(),
90                format!("failed to read directory at `{}`: {e}", path.display()),
91            )
92        })?;
93    Ok(st.try_filter_map(|e| async move {
94        let name = e.file_name();
95        if !is_wit(&name) {
96            trace!("{} is not a WIT definition, skip", name.to_string_lossy());
97            return Ok(None);
98        }
99        if e.file_type().await?.is_dir() {
100            trace!("{} is a directory, skip", name.to_string_lossy());
101            return Ok(None);
102        }
103        Ok(Some(name))
104    }))
105}
106
107/// Copies all WIT definitions from directory at `src` to `dst` creating `dst` directory, if it does not exist.
108#[instrument(level = "trace", skip(src, dst))]
109async fn install_wits(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io::Result<()> {
110    let src = src.as_ref();
111    let dst = dst.as_ref();
112    recreate_dir(dst).await?;
113    read_wits(src)
114        .await?
115        .try_for_each_concurrent(None, |name| async {
116            let src = src.join(&name);
117            let dst = dst.join(name);
118            fs::copy(&src, &dst)
119                .await
120                .map(|_| trace!("copied `{}` to `{}`", src.display(), dst.display()))
121                .map_err(|e| {
122                    std::io::Error::new(
123                        e.kind(),
124                        format!(
125                            "failed to copy `{}` to `{}`: {e}",
126                            src.display(),
127                            dst.display()
128                        ),
129                    )
130                })
131        })
132        .await
133}
134
135/// Copies all WIT files from directory at `src` to `dst` and returns a vector identifiers of all copied
136/// transitive dependencies.
137#[instrument(level = "trace", skip(src, dst, skip_deps))]
138async fn copy_wits(
139    src: impl AsRef<Path>,
140    dst: impl AsRef<Path>,
141    skip_deps: &HashSet<Identifier>,
142) -> std::io::Result<HashMap<Identifier, PathBuf>> {
143    let src = src.as_ref();
144    let deps = src.join("deps");
145    let dst = dst.as_ref();
146    try_join!(install_wits(src, dst), async {
147        match (dst.parent(), fs::read_dir(&deps).await) {
148            (Some(base), Ok(dir)) => {
149                ReadDirStream::new(dir)
150                    .try_filter_map(|e| async move {
151                        let name = e.file_name();
152                        let Some(id) = name.to_str().map(Identifier::from) else {
153                            return Ok(None);
154                        };
155                        if skip_deps.contains(&id) {
156                            return Ok(None);
157                        }
158                        let ft = e.file_type().await?;
159                        if !(ft.is_dir()
160                            || ft.is_symlink() && fs::metadata(e.path()).await?.is_dir())
161                        {
162                            return Ok(None);
163                        }
164                        Ok(Some(id))
165                    })
166                    .and_then(|id| async {
167                        let dst = base.join(&id);
168                        install_wits(deps.join(&id), &dst).await?;
169                        Ok((id, dst))
170                    })
171                    .try_collect()
172                    .await
173            }
174            (None, _) => Ok(HashMap::default()),
175            (_, Err(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(HashMap::default()),
176            (_, Err(e)) => Err(std::io::Error::new(
177                e.kind(),
178                format!("failed to read directory at `{}`: {e}", deps.display()),
179            )),
180        }
181    })
182    .map(|((), ids)| ids)
183}
184
185/// Unpacks all WIT interfaces found within `wit` subtree of a tar archive read from `tar` to
186/// `dst` and returns a [`HashMap`] of all unpacked transitive dependency identifiers.
187///
188/// # Errors
189///
190/// Returns and error if the operation fails
191#[instrument(level = "trace", skip(tar, dst, skip_deps))]
192pub async fn untar(
193    tar: impl AsyncRead + Unpin,
194    dst: impl AsRef<Path>,
195    skip_deps: &HashSet<Identifier>,
196    prefix: &str,
197) -> std::io::Result<HashMap<Identifier, PathBuf>> {
198    use std::io::{Error, Result};
199
200    async fn unpack(e: &mut async_tar::Entry<impl Unpin + AsyncRead>, dst: &Path) -> Result<()> {
201        e.unpack(dst).await.map_err(|e: Error| {
202            Error::new(
203                e.kind(),
204                format!("failed to unpack `{}`: {e}", dst.display()),
205            )
206        })?;
207        trace!("unpacked `{}`", dst.display());
208        Ok(())
209    }
210
211    let dst = dst.as_ref();
212    recreate_dir(dst).await?;
213    async_tar::Archive::new(tar)
214        .entries()
215        .map_err(|e| Error::new(e.kind(), format!("failed to unpack archive metadata: {e}")))?
216        .try_fold(HashMap::default(), |mut untared, mut e| async move {
217            let path = e
218                .path()
219                .map_err(|e| Error::new(e.kind(), format!("failed to query entry path: {e}")))?;
220            let Ok(path) = path.strip_prefix(prefix) else {
221                return Ok(untared);
222            };
223            let mut path = path.iter();
224            match array::from_fn::<_, 6, _>(|_| path.next().and_then(OsStr::to_str)) {
225                [Some(name), None, ..]
226                | [Some("wit"), Some(name), None, ..]
227                | [Some(_), Some("wit"), Some(name), None, ..]
228                    if is_wit(name) =>
229                {
230                    let dst = dst.join(name);
231                    unpack(&mut e, &dst).await?;
232                    Ok(untared)
233                }
234                [Some("deps"), Some(id), Some(name), None, ..]
235                | [Some("wit"), Some("deps"), Some(id), Some(name), None, ..]
236                | [Some(_), Some("wit"), Some("deps"), Some(id), Some(name), None]
237                    if !skip_deps.contains(id) && is_wit(name) =>
238                {
239                    let id = Identifier::from(id);
240                    if let Some(base) = dst.parent() {
241                        let dst = base.join(&id);
242                        if !untared.contains_key(&id) {
243                            recreate_dir(&dst).await?;
244                        }
245                        let wit = dst.join(name);
246                        unpack(&mut e, &wit).await?;
247                        untared.insert(id, dst);
248                        Ok(untared)
249                    } else {
250                        Ok(untared)
251                    }
252                }
253                _ => Ok(untared),
254            }
255        })
256        .await
257}
258
259/// Packages path into a `wit` subtree in deterministic `tar` archive and writes it to `dst`.
260///
261/// # Errors
262///
263/// Returns and error if the operation fails
264#[instrument(level = "trace", skip(path, dst))]
265pub async fn tar<T>(path: impl AsRef<Path>, dst: T) -> std::io::Result<T>
266where
267    T: AsyncWrite + Sync + Send + Unpin,
268{
269    let path = path.as_ref();
270    let mut tar = async_tar::Builder::new(dst);
271    tar.mode(async_tar::HeaderMode::Deterministic);
272    let res = async {
273        for name in read_wits(path).await?.try_collect::<BTreeSet<_>>().await? {
274            tar.append_path_with_name(path.join(&name), Path::new("wit").join(name))
275                .await?;
276        }
277        std::io::Result::Ok(())
278    }
279    .await;
280    if res.is_err() {
281        // Finalize the builder to avoid a panic on drop.
282        let _ = tar.finish().await;
283    }
284    res?;
285    tar.into_inner().await
286}
287
288fn cache() -> Option<impl Cache> {
289    LocalCache::cache_dir().map(|cache| {
290        debug!("using cache at `{cache}`");
291        cache
292    })
293}
294
295/// Given a TOML-encoded manifest and optional TOML-encoded lock, ensures that the path pointed to by
296/// `deps` is in sync with the manifest and lock. This is a potentially destructive operation!
297/// Returns a TOML-encoded lock if the lock passed to this function was either `None` or out-of-sync.
298///
299/// # Errors
300///
301/// Returns an error if anything in the pipeline fails
302#[instrument(level = "trace", skip(at, manifest, lock, deps))]
303pub async fn lock(
304    at: Option<impl AsRef<Path>>,
305    manifest: impl AsRef<str>,
306    lock: Option<impl AsRef<str>>,
307    deps: impl AsRef<Path>,
308) -> anyhow::Result<Option<String>> {
309    let manifest: Manifest =
310        toml::from_str(manifest.as_ref()).context("failed to decode manifest")?;
311
312    let old_lock = lock
313        .as_ref()
314        .map(AsRef::as_ref)
315        .map(toml::from_str)
316        .transpose()
317        .context("failed to decode lock")?;
318
319    let deps = deps.as_ref();
320    let lock = manifest
321        .lock(at, deps, old_lock.as_ref(), cache().as_ref())
322        .await
323        .with_context(|| format!("failed to lock deps to `{}`", deps.display()))?;
324    match old_lock {
325        Some(old_lock) if lock == old_lock => Ok(None),
326        _ => toml::to_string(&lock)
327            .map(Some)
328            .context("failed to encode lock"),
329    }
330}
331
332/// Given a TOML-encoded manifest, ensures that the path pointed to by
333/// `deps` is in sync with the manifest. This is a potentially destructive operation!
334/// Returns a TOML-encoded lock on success.
335///
336/// # Errors
337///
338/// Returns an error if anything in the pipeline fails
339#[instrument(level = "trace", skip(at, manifest, deps))]
340pub async fn update(
341    at: Option<impl AsRef<Path>>,
342    manifest: impl AsRef<str>,
343    deps: impl AsRef<Path>,
344) -> anyhow::Result<String> {
345    let manifest: Manifest =
346        toml::from_str(manifest.as_ref()).context("failed to decode manifest")?;
347
348    let deps = deps.as_ref();
349    let lock = manifest
350        .lock(at, deps, None, cache().map(WriteCache).as_ref())
351        .await
352        .with_context(|| format!("failed to lock deps to `{}`", deps.display()))?;
353    toml::to_string(&lock).context("failed to encode lock")
354}
355
356async fn read_manifest_string(path: impl AsRef<Path>) -> std::io::Result<String> {
357    let path = path.as_ref();
358    fs::read_to_string(&path).await.map_err(|e| {
359        std::io::Error::new(
360            e.kind(),
361            format!("failed to read manifest at `{}`: {e}", path.display()),
362        )
363    })
364}
365
366async fn write_lock(path: impl AsRef<Path>, buf: impl AsRef<[u8]>) -> std::io::Result<()> {
367    let path = path.as_ref();
368    if let Some(parent) = path.parent() {
369        fs::create_dir_all(parent).await.map_err(|e| {
370            std::io::Error::new(
371                e.kind(),
372                format!(
373                    "failed to create lock parent directory `{}`: {e}",
374                    parent.display()
375                ),
376            )
377        })?;
378    }
379    fs::write(&path, &buf).await.map_err(|e| {
380        std::io::Error::new(
381            e.kind(),
382            format!("failed to write lock to `{}`: {e}", path.display()),
383        )
384    })
385}
386
387/// Like [lock](self::lock()), but reads the manifest at `manifest_path` and reads/writes the lock at `lock_path`.
388///
389/// Returns `true` if the lock was updated and `false` otherwise.
390///
391/// # Errors
392///
393/// Returns an error if anything in the pipeline fails
394#[instrument(level = "trace", skip(manifest_path, lock_path, deps))]
395pub async fn lock_path(
396    manifest_path: impl AsRef<Path>,
397    lock_path: impl AsRef<Path>,
398    deps: impl AsRef<Path>,
399) -> anyhow::Result<bool> {
400    let manifest_path = manifest_path.as_ref();
401    let lock_path = lock_path.as_ref();
402    let (manifest, lock) = try_join!(
403        read_manifest_string(manifest_path),
404        fs::read_to_string(&lock_path).map(|res| match res {
405            Ok(lock) => Ok(Some(lock)),
406            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
407            Err(e) => Err(std::io::Error::new(
408                e.kind(),
409                format!("failed to read lock at `{}`: {e}", lock_path.display())
410            )),
411        }),
412    )?;
413    if let Some(lock) = self::lock(manifest_path.parent(), manifest, lock, deps)
414        .await
415        .context("failed to lock dependencies")?
416    {
417        write_lock(lock_path, lock).await?;
418        Ok(true)
419    } else {
420        Ok(false)
421    }
422}
423
424/// Like [update](self::update()), but reads the manifest at `manifest_path` and writes the lock at `lock_path`.
425///
426/// # Errors
427///
428/// Returns an error if anything in the pipeline fails
429#[instrument(level = "trace", skip(manifest_path, lock_path, deps))]
430pub async fn update_path(
431    manifest_path: impl AsRef<Path>,
432    lock_path: impl AsRef<Path>,
433    deps: impl AsRef<Path>,
434) -> anyhow::Result<()> {
435    let manifest_path = manifest_path.as_ref();
436    let manifest = read_manifest_string(manifest_path).await?;
437    let lock = self::update(manifest_path.parent(), manifest, deps)
438        .await
439        .context("failed to lock dependencies")?;
440    write_lock(lock_path, lock).await?;
441    Ok(())
442}
443
444/// Asynchronously ensure dependency manifest, lock and dependencies are in sync.
445/// This must run within a [tokio] context.
446#[macro_export]
447macro_rules! lock {
448    () => {
449        $crate::lock!("wit")
450    };
451    ($dir:literal $(,)?) => {
452        async {
453            use $crate::tokio::fs;
454
455            use std::io::{Error, ErrorKind};
456
457            let lock = match fs::read_to_string(concat!($dir, "/deps.lock")).await {
458                Ok(lock) => Some(lock),
459                Err(e) if e.kind() == ErrorKind::NotFound => None,
460                Err(e) => {
461                    return Err(Error::new(
462                        e.kind(),
463                        format!(
464                            "failed to read lock at `{}`: {e}",
465                            concat!($dir, "/deps.lock")
466                        ),
467                    ))
468                }
469            };
470            match $crate::lock(
471                Some($dir),
472                include_str!(concat!($dir, "/deps.toml")),
473                lock,
474                concat!($dir, "/deps"),
475            )
476            .await
477            {
478                Ok(Some(lock)) => fs::write(concat!($dir, "/deps.lock"), lock)
479                    .await
480                    .map_err(|e| {
481                        Error::new(
482                            e.kind(),
483                            format!(
484                                "failed to write lock at `{}`: {e}",
485                                concat!($dir, "/deps.lock")
486                            ),
487                        )
488                    }),
489                Ok(None) => Ok(()),
490                Err(e) => Err(Error::new(ErrorKind::Other, e)),
491            }
492        }
493    };
494}
495
496#[cfg(feature = "sync")]
497/// Synchronously ensure dependency manifest, lock and dependencies are in sync.
498#[macro_export]
499macro_rules! lock_sync {
500    ($($args:tt)*) => {
501        $crate::tokio::runtime::Builder::new_multi_thread()
502            .thread_name("wit-deps/lock_sync")
503            .enable_io()
504            .enable_time()
505            .build()?
506            .block_on($crate::lock!($($args)*))
507    };
508}
wit_deps/lib.rs

wit_deps/
lib.rs
