wit_deps/
lib.rs

1//! WIT dependency management core library
2
3#![forbid(clippy::unwrap_used)]
4#![warn(missing_docs)]
5
6mod cache;
7mod digest;
8mod lock;
9mod manifest;
10
11pub use cache::{Cache, Local as LocalCache, Write as WriteCache};
12pub use digest::{Digest, Reader as DigestReader, Writer as DigestWriter};
13pub use lock::{Entry as LockEntry, EntrySource as LockEntrySource, Lock};
14pub use manifest::{Entry as ManifestEntry, Manifest};
15
16pub use futures;
17pub use tokio;
18
19use std::collections::{BTreeSet, HashMap, HashSet};
20use std::ffi::{OsStr, OsString};
21use std::path::{Path, PathBuf};
22
23use anyhow::Context;
24use futures::{try_join, AsyncRead, AsyncWrite, FutureExt, Stream, TryStreamExt};
25use tokio::fs;
26use tokio_stream::wrappers::ReadDirStream;
27use tracing::{debug, instrument, trace};
28
29/// WIT dependency identifier
30pub type Identifier = String;
31// TODO: Introduce a rich type with name validation
32//#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq)]
33//pub struct Identifier(String);
34
35fn is_wit(path: impl AsRef<Path>) -> bool {
36    path.as_ref()
37        .extension()
38        .is_some_and(|ext| ext.eq_ignore_ascii_case("wit"))
39}
40
41#[instrument(level = "trace", skip(path))]
42async fn remove_dir_all(path: impl AsRef<Path>) -> std::io::Result<()> {
43    let path = path.as_ref();
44    match fs::remove_dir_all(path).await {
45        Ok(()) => {
46            trace!("removed `{}`", path.display());
47            Ok(())
48        }
49        Err(e) => Err(std::io::Error::new(
50            e.kind(),
51            format!("failed to remove `{}`: {e}", path.display()),
52        )),
53    }
54}
55
56#[instrument(level = "trace", skip(path))]
57async fn recreate_dir(path: impl AsRef<Path>) -> std::io::Result<()> {
58    let path = path.as_ref();
59    match remove_dir_all(path).await {
60        Ok(()) => {}
61        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
62        Err(e) => return Err(e),
63    }
64    fs::create_dir_all(path)
65        .await
66        .map(|()| trace!("recreated `{}`", path.display()))
67        .map_err(|e| {
68            std::io::Error::new(
69                e.kind(),
70                format!("failed to create `{}`: {e}", path.display()),
71            )
72        })
73}
74
75/// Returns a stream of WIT file names within a directory at `path`
76#[instrument(level = "trace", skip(path))]
77async fn read_wits(
78    path: impl AsRef<Path>,
79) -> std::io::Result<impl Stream<Item = std::io::Result<OsString>>> {
80    let path = path.as_ref();
81    let st = fs::read_dir(path)
82        .await
83        .map(ReadDirStream::new)
84        .map_err(|e| {
85            std::io::Error::new(
86                e.kind(),
87                format!("failed to read directory at `{}`: {e}", path.display()),
88            )
89        })?;
90    Ok(st.try_filter_map(|e| async move {
91        let name = e.file_name();
92        if !is_wit(&name) {
93            trace!("{} is not a WIT definition, skip", name.to_string_lossy());
94            return Ok(None);
95        }
96        if e.file_type().await?.is_dir() {
97            trace!("{} is a directory, skip", name.to_string_lossy());
98            return Ok(None);
99        }
100        Ok(Some(name))
101    }))
102}
103
104/// Copies all WIT definitions from directory at `src` to `dst` creating `dst` directory, if it does not exist.
105#[instrument(level = "trace", skip(src, dst))]
106async fn install_wits(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io::Result<()> {
107    let src = src.as_ref();
108    let dst = dst.as_ref();
109    recreate_dir(dst).await?;
110    read_wits(src)
111        .await?
112        .try_for_each_concurrent(None, |name| async {
113            let src = src.join(&name);
114            let dst = dst.join(name);
115            fs::copy(&src, &dst)
116                .await
117                .map(|_| trace!("copied `{}` to `{}`", src.display(), dst.display()))
118                .map_err(|e| {
119                    std::io::Error::new(
120                        e.kind(),
121                        format!(
122                            "failed to copy `{}` to `{}`: {e}",
123                            src.display(),
124                            dst.display()
125                        ),
126                    )
127                })
128        })
129        .await
130}
131
132/// Copies all WIT files from directory at `src` to `dst` and returns a vector identifiers of all copied
133/// transitive dependencies.
134#[instrument(level = "trace", skip(src, dst, skip_deps))]
135async fn copy_wits(
136    src: impl AsRef<Path>,
137    dst: impl AsRef<Path>,
138    skip_deps: &HashSet<Identifier>,
139) -> std::io::Result<HashMap<Identifier, PathBuf>> {
140    let src = src.as_ref();
141    let deps = src.join("deps");
142    let dst = dst.as_ref();
143    try_join!(install_wits(src, dst), async {
144        match (dst.parent(), fs::read_dir(&deps).await) {
145            (Some(base), Ok(dir)) => {
146                ReadDirStream::new(dir)
147                    .try_filter_map(|e| async move {
148                        let name = e.file_name();
149                        let Some(id) = name.to_str().map(Identifier::from) else {
150                            return Ok(None);
151                        };
152                        if skip_deps.contains(&id) {
153                            return Ok(None);
154                        }
155                        let ft = e.file_type().await?;
156                        if !(ft.is_dir()
157                            || ft.is_symlink() && fs::metadata(e.path()).await?.is_dir())
158                        {
159                            return Ok(None);
160                        }
161                        Ok(Some(id))
162                    })
163                    .and_then(|id| async {
164                        let dst = base.join(&id);
165                        install_wits(deps.join(&id), &dst).await?;
166                        Ok((id, dst))
167                    })
168                    .try_collect()
169                    .await
170            }
171            (None, _) => Ok(HashMap::default()),
172            (_, Err(e)) if e.kind() == std::io::ErrorKind::NotFound => Ok(HashMap::default()),
173            (_, Err(e)) => Err(std::io::Error::new(
174                e.kind(),
175                format!("failed to read directory at `{}`: {e}", deps.display()),
176            )),
177        }
178    })
179    .map(|((), ids)| ids)
180}
181
182/// Unpacks all WIT interfaces found within `wit` subtree of a tar archive read from `tar` to
183/// `dst` and returns a [`HashMap`] of all unpacked transitive dependency identifiers.
184///
185/// # Errors
186///
187/// Returns and error if the operation fails
188#[instrument(level = "trace", skip(tar, dst, skip_deps))]
189pub async fn untar(
190    tar: impl AsyncRead + Unpin,
191    dst: impl AsRef<Path>,
192    skip_deps: &HashSet<Identifier>,
193    subdir: &str,
194) -> std::io::Result<HashMap<Identifier, PathBuf>> {
195    use std::io::{Error, Result};
196
197    async fn unpack(e: &mut async_tar::Entry<impl Unpin + AsyncRead>, dst: &Path) -> Result<()> {
198        e.unpack(dst).await.map_err(|e| {
199            Error::new(
200                e.kind(),
201                format!("failed to unpack `{}`: {e}", dst.display()),
202            )
203        })?;
204        trace!("unpacked `{}`", dst.display());
205        Ok(())
206    }
207
208    let dst = dst.as_ref();
209    recreate_dir(dst).await?;
210    async_tar::Archive::new(tar)
211        .entries()
212        .map_err(|e| Error::new(e.kind(), format!("failed to unpack archive metadata: {e}")))?
213        .try_fold(HashMap::default(), |mut untared, mut e| async move {
214            let path = e
215                .path()
216                .map_err(|e| Error::new(e.kind(), format!("failed to query entry path: {e}")))?;
217            let mut path = path.into_iter().map(OsStr::to_str);
218            match (
219                path.next(),
220                path.next(),
221                path.next(),
222                path.next(),
223                path.next(),
224            ) {
225                (Some(Some(name)), None, None, None, None)
226                | (Some(_), Some(Some(name)), None, None, None)
227                    if is_wit(name) && subdir.is_empty() =>
228                {
229                    let dst = dst.join(name);
230                    unpack(&mut e, &dst).await?;
231                    Ok(untared)
232                }
233                (Some(Some(dir)), Some(Some(name)), None, None, None)
234                | (Some(_), Some(Some(dir)), Some(Some(name)), None, None)
235                    if is_wit(name) && dir == subdir =>
236                {
237                    let dst = dst.join(name);
238                    unpack(&mut e, &dst).await?;
239                    Ok(untared)
240                }
241                (Some(Some("deps")), Some(Some(id)), Some(Some(name)), None, None)
242                | (Some(_), Some(Some("deps")), Some(Some(id)), Some(Some(name)), None)
243                    if !skip_deps.contains(id) && is_wit(name) && subdir.is_empty() =>
244                {
245                    let id = Identifier::from(id);
246                    if let Some(base) = dst.parent() {
247                        let dst = base.join(&id);
248                        if !untared.contains_key(&id) {
249                            recreate_dir(&dst).await?;
250                        }
251                        let wit = dst.join(name);
252                        unpack(&mut e, &wit).await?;
253                        untared.insert(id, dst);
254                        Ok(untared)
255                    } else {
256                        Ok(untared)
257                    }
258                }
259                (Some(Some(dir)), Some(Some("deps")), Some(Some(id)), Some(Some(name)), None)
260                | (
261                    Some(_),
262                    Some(Some(dir)),
263                    Some(Some("deps")),
264                    Some(Some(id)),
265                    Some(Some(name)),
266                ) if !skip_deps.contains(id) && is_wit(name) && dir == subdir => {
267                    let id = Identifier::from(id);
268                    if let Some(base) = dst.parent() {
269                        let dst = base.join(&id);
270                        if !untared.contains_key(&id) {
271                            recreate_dir(&dst).await?;
272                        }
273                        let wit = dst.join(name);
274                        unpack(&mut e, &wit).await?;
275                        untared.insert(id, dst);
276                        Ok(untared)
277                    } else {
278                        Ok(untared)
279                    }
280                }
281                _ => Ok(untared),
282            }
283        })
284        .await
285}
286
287/// Packages path into a `wit` subtree in deterministic `tar` archive and writes it to `dst`.
288///
289/// # Errors
290///
291/// Returns and error if the operation fails
292#[instrument(level = "trace", skip(path, dst))]
293pub async fn tar<T>(path: impl AsRef<Path>, dst: T) -> std::io::Result<T>
294where
295    T: AsyncWrite + Sync + Send + Unpin,
296{
297    let path = path.as_ref();
298    let mut tar = async_tar::Builder::new(dst);
299    tar.mode(async_tar::HeaderMode::Deterministic);
300    for name in read_wits(path).await?.try_collect::<BTreeSet<_>>().await? {
301        tar.append_path_with_name(path.join(&name), Path::new("wit").join(name))
302            .await?;
303    }
304    tar.into_inner().await
305}
306
307fn cache() -> Option<impl Cache> {
308    LocalCache::cache_dir().map(|cache| {
309        debug!("using cache at `{cache}`");
310        cache
311    })
312}
313
314/// Given a TOML-encoded manifest and optional TOML-encoded lock, ensures that the path pointed to by
315/// `deps` is in sync with the manifest and lock. This is a potentially destructive operation!
316/// Returns a TOML-encoded lock if the lock passed to this function was either `None` or out-of-sync.
317///
318/// # Errors
319///
320/// Returns an error if anything in the pipeline fails
321#[instrument(level = "trace", skip(at, manifest, lock, deps))]
322pub async fn lock(
323    at: Option<impl AsRef<Path>>,
324    manifest: impl AsRef<str>,
325    lock: Option<impl AsRef<str>>,
326    deps: impl AsRef<Path>,
327) -> anyhow::Result<Option<String>> {
328    let manifest: Manifest =
329        toml::from_str(manifest.as_ref()).context("failed to decode manifest")?;
330
331    let old_lock = lock
332        .as_ref()
333        .map(AsRef::as_ref)
334        .map(toml::from_str)
335        .transpose()
336        .context("failed to decode lock")?;
337
338    let deps = deps.as_ref();
339    let lock = manifest
340        .lock(at, deps, old_lock.as_ref(), cache().as_ref())
341        .await
342        .with_context(|| format!("failed to lock deps to `{}`", deps.display()))?;
343    match old_lock {
344        Some(old_lock) if lock == old_lock => Ok(None),
345        _ => toml::to_string(&lock)
346            .map(Some)
347            .context("failed to encode lock"),
348    }
349}
350
351/// Given a TOML-encoded manifest, ensures that the path pointed to by
352/// `deps` is in sync with the manifest. This is a potentially destructive operation!
353/// Returns a TOML-encoded lock on success.
354///
355/// # Errors
356///
357/// Returns an error if anything in the pipeline fails
358#[instrument(level = "trace", skip(at, manifest, deps))]
359pub async fn update(
360    at: Option<impl AsRef<Path>>,
361    manifest: impl AsRef<str>,
362    deps: impl AsRef<Path>,
363) -> anyhow::Result<String> {
364    let manifest: Manifest =
365        toml::from_str(manifest.as_ref()).context("failed to decode manifest")?;
366
367    let deps = deps.as_ref();
368    let lock = manifest
369        .lock(at, deps, None, cache().map(WriteCache).as_ref())
370        .await
371        .with_context(|| format!("failed to lock deps to `{}`", deps.display()))?;
372    toml::to_string(&lock).context("failed to encode lock")
373}
374
375async fn read_manifest_string(path: impl AsRef<Path>) -> std::io::Result<String> {
376    let path = path.as_ref();
377    fs::read_to_string(&path).await.map_err(|e| {
378        std::io::Error::new(
379            e.kind(),
380            format!("failed to read manifest at `{}`: {e}", path.display()),
381        )
382    })
383}
384
385async fn write_lock(path: impl AsRef<Path>, buf: impl AsRef<[u8]>) -> std::io::Result<()> {
386    let path = path.as_ref();
387    if let Some(parent) = path.parent() {
388        fs::create_dir_all(parent).await.map_err(|e| {
389            std::io::Error::new(
390                e.kind(),
391                format!(
392                    "failed to create lock parent directory `{}`: {e}",
393                    parent.display()
394                ),
395            )
396        })?;
397    }
398    fs::write(&path, &buf).await.map_err(|e| {
399        std::io::Error::new(
400            e.kind(),
401            format!("failed to write lock to `{}`: {e}", path.display()),
402        )
403    })
404}
405
406/// Like [lock](self::lock()), but reads the manifest at `manifest_path` and reads/writes the lock at `lock_path`.
407///
408/// Returns `true` if the lock was updated and `false` otherwise.
409///
410/// # Errors
411///
412/// Returns an error if anything in the pipeline fails
413#[instrument(level = "trace", skip(manifest_path, lock_path, deps))]
414pub async fn lock_path(
415    manifest_path: impl AsRef<Path>,
416    lock_path: impl AsRef<Path>,
417    deps: impl AsRef<Path>,
418) -> anyhow::Result<bool> {
419    let manifest_path = manifest_path.as_ref();
420    let lock_path = lock_path.as_ref();
421    let (manifest, lock) = try_join!(
422        read_manifest_string(manifest_path),
423        fs::read_to_string(&lock_path).map(|res| match res {
424            Ok(lock) => Ok(Some(lock)),
425            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
426            Err(e) => Err(std::io::Error::new(
427                e.kind(),
428                format!("failed to read lock at `{}`: {e}", lock_path.display())
429            )),
430        }),
431    )?;
432    if let Some(lock) = self::lock(manifest_path.parent(), manifest, lock, deps)
433        .await
434        .context("failed to lock dependencies")?
435    {
436        write_lock(lock_path, lock).await?;
437        Ok(true)
438    } else {
439        Ok(false)
440    }
441}
442
443/// Like [update](self::update()), but reads the manifest at `manifest_path` and writes the lock at `lock_path`.
444///
445/// # Errors
446///
447/// Returns an error if anything in the pipeline fails
448#[instrument(level = "trace", skip(manifest_path, lock_path, deps))]
449pub async fn update_path(
450    manifest_path: impl AsRef<Path>,
451    lock_path: impl AsRef<Path>,
452    deps: impl AsRef<Path>,
453) -> anyhow::Result<()> {
454    let manifest_path = manifest_path.as_ref();
455    let manifest = read_manifest_string(manifest_path).await?;
456    let lock = self::update(manifest_path.parent(), manifest, deps)
457        .await
458        .context("failed to lock dependencies")?;
459    write_lock(lock_path, lock).await?;
460    Ok(())
461}
462
463/// Asynchronously ensure dependency manifest, lock and dependencies are in sync.
464/// This must run within a [tokio] context.
465#[macro_export]
466macro_rules! lock {
467    () => {
468        $crate::lock!("wit")
469    };
470    ($dir:literal $(,)?) => {
471        async {
472            use $crate::tokio::fs;
473
474            use std::io::{Error, ErrorKind};
475
476            let lock = match fs::read_to_string(concat!($dir, "/deps.lock")).await {
477                Ok(lock) => Some(lock),
478                Err(e) if e.kind() == ErrorKind::NotFound => None,
479                Err(e) => {
480                    return Err(Error::new(
481                        e.kind(),
482                        format!(
483                            "failed to read lock at `{}`: {e}",
484                            concat!($dir, "/deps.lock")
485                        ),
486                    ))
487                }
488            };
489            match $crate::lock(
490                Some($dir),
491                include_str!(concat!($dir, "/deps.toml")),
492                lock,
493                concat!($dir, "/deps"),
494            )
495            .await
496            {
497                Ok(Some(lock)) => fs::write(concat!($dir, "/deps.lock"), lock)
498                    .await
499                    .map_err(|e| {
500                        Error::new(
501                            e.kind(),
502                            format!(
503                                "failed to write lock at `{}`: {e}",
504                                concat!($dir, "/deps.lock")
505                            ),
506                        )
507                    }),
508                Ok(None) => Ok(()),
509                Err(e) => Err(Error::new(ErrorKind::Other, e)),
510            }
511        }
512    };
513}
514
515#[cfg(feature = "sync")]
516/// Synchronously ensure dependency manifest, lock and dependencies are in sync.
517#[macro_export]
518macro_rules! lock_sync {
519    ($($args:tt)*) => {
520        $crate::tokio::runtime::Builder::new_multi_thread()
521            .thread_name("wit-deps/lock_sync")
522            .enable_io()
523            .enable_time()
524            .build()?
525            .block_on($crate::lock!($($args)*))
526    };
527}
wit_deps/lib.rs

wit_deps/
lib.rs
